Mercurial > hg > Members > aka > jupyter_CbC_kernel
annotate jupyter_c_kernel/kernel.py @ 54:7857f8eea835
Add args magic to provide cli-args to user program
author | Ben Spoor <ben.spoor@altran.com> |
---|---|
date | Thu, 13 Apr 2017 08:09:16 +0000 |
parents | 0ef931211f77 |
children | 40c903dde893 |
rev | line source |
---|---|
39
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
1 from queue import Queue |
33 | 2 from threading import Thread |
3 | |
0 | 4 from ipykernel.kernelbase import Kernel |
54
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
5 import re |
0 | 6 import subprocess |
7 import tempfile | |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
8 import os |
36
8acbab5a9f21
Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
35
diff
changeset
|
9 import os.path as path |
34 | 10 |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
11 |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
12 class RealTimeSubprocess(subprocess.Popen): |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
13 """ |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
14 A subprocess that allows to read its stdout and stderr in real time |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
15 """ |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
16 |
34 | 17 def __init__(self, cmd, write_to_stdout, write_to_stderr): |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
18 """ |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
19 :param cmd: the command to execute |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
20 :param write_to_stdout: a callable that will be called with chunks of data from stdout |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
21 :param write_to_stderr: a callable that will be called with chunks of data from stderr |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
22 """ |
34 | 23 self._write_to_stdout = write_to_stdout |
24 self._write_to_stderr = write_to_stderr | |
25 | |
37
aac47bc07111
Fix subprocess bufsize & master compilation
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
36
diff
changeset
|
26 super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) |
34 | 27 |
28 self._stdout_queue = Queue() | |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
29 self._stdout_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stdout, self._stdout_queue)) |
34 | 30 self._stdout_thread.daemon = True |
31 self._stdout_thread.start() | |
32 | |
33 self._stderr_queue = Queue() | |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
34 self._stderr_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stderr, self._stderr_queue)) |
34 | 35 self._stderr_thread.daemon = True |
36 self._stderr_thread.start() | |
37 | |
38 @staticmethod | |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
39 def _enqueue_output(stream, queue): |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
40 """ |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
41 Add chunks of data from a stream to a queue until the stream is empty. |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
42 """ |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
43 for line in iter(lambda: stream.read(4096), b''): |
34 | 44 queue.put(line) |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
45 stream.close() |
34 | 46 |
35 | 47 def write_contents(self): |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
48 """ |
39
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
49 Write the available content from stdin and stderr where specified when the instance was created |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
50 :return: |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
51 """ |
39
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
52 |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
53 def read_all_from_queue(queue): |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
54 res = b'' |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
55 size = queue.qsize() |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
56 while size != 0: |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
57 res += queue.get_nowait() |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
58 size -= 1 |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
59 return res |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
60 |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
61 stdout_contents = read_all_from_queue(self._stdout_queue) |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
62 if stdout_contents: |
34 | 63 self._write_to_stdout(stdout_contents) |
39
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
64 stderr_contents = read_all_from_queue(self._stderr_queue) |
fecdf8733f3b
Make sure to empty the queue so no message should be left over in it
Brendan Rius <brendan@omixy.com>
parents:
38
diff
changeset
|
65 if stderr_contents: |
34 | 66 self._write_to_stderr(stderr_contents) |
0 | 67 |
68 | |
69 class CKernel(Kernel): | |
20 | 70 implementation = 'jupyter_c_kernel' |
0 | 71 implementation_version = '1.0' |
72 language = 'c' | |
73 language_version = 'C11' | |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
74 language_info = {'name': 'c', |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
75 'mimetype': 'text/plain', |
50
a98647258349
solve error “traitlets.traitlets.TraitError: FileExtension trait 'file_extension' does not begin with a dot: ‘c’” when exporting to script with nbconvert
Davide Bortolami <d@fermiumlabs.com>
parents:
39
diff
changeset
|
76 'file_extension': '.c'} |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
77 banner = "C kernel.\n" \ |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
78 "Uses gcc, compiles in C11, and creates source code files and executables in temporary folder.\n" |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
79 |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
80 def __init__(self, *args, **kwargs): |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
81 super(CKernel, self).__init__(*args, **kwargs) |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
82 self.files = [] |
36
8acbab5a9f21
Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
35
diff
changeset
|
83 mastertemp = tempfile.mkstemp(suffix='.out') |
8acbab5a9f21
Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
35
diff
changeset
|
84 os.close(mastertemp[0]) |
8acbab5a9f21
Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
35
diff
changeset
|
85 self.master_path = mastertemp[1] |
8acbab5a9f21
Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
35
diff
changeset
|
86 filepath = path.join(path.dirname(path.realpath(__file__)), '..', 'resources', 'master.c') |
37
aac47bc07111
Fix subprocess bufsize & master compilation
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
36
diff
changeset
|
87 subprocess.call(['gcc', filepath, '-std=c11', '-rdynamic', '-ldl', '-o', self.master_path]) |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
88 |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
89 def cleanup_files(self): |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
90 """Remove all the temporary files created by the kernel""" |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
91 for file in self.files: |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
92 os.remove(file) |
36
8acbab5a9f21
Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents:
35
diff
changeset
|
93 os.remove(self.master_path) |
0 | 94 |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
95 def new_temp_file(self, **kwargs): |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
96 """Create a new temp file to be deleted when the kernel shuts down""" |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
97 # We don't want the file to be deleted when closed, but only when the kernel stops |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
98 kwargs['delete'] = False |
9 | 99 kwargs['mode'] = 'w' |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
100 file = tempfile.NamedTemporaryFile(**kwargs) |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
101 self.files.append(file.name) |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
102 return file |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
103 |
34 | 104 def _write_to_stdout(self, contents): |
105 self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': contents}) | |
33 | 106 |
34 | 107 def _write_to_stderr(self, contents): |
108 self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': contents}) | |
33 | 109 |
34 | 110 def create_jupyter_subprocess(self, cmd): |
38
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
111 return RealTimeSubprocess(cmd, |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
112 lambda contents: self._write_to_stdout(contents.decode()), |
cb587f21c8bb
Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents:
37
diff
changeset
|
113 lambda contents: self._write_to_stderr(contents.decode())) |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
114 |
52
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
115 def compile_with_gcc(self, source_filename, binary_filename, cflags=None, ldflags=None): |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
116 cflags = ['-std=c11', '-fPIC', '-shared', '-rdynamic'] + cflags |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
117 args = ['gcc', source_filename] + cflags + ['-o', binary_filename] + ldflags |
34 | 118 return self.create_jupyter_subprocess(args) |
0 | 119 |
52
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
120 def _filter_magics(self, code): |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
121 |
54
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
122 magics = {'cflags': [], |
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
123 'ldflags': [], |
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
124 'args': []} |
52
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
125 |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
126 for line in code.splitlines(): |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
127 if line.startswith('//%'): |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
128 key, value = line[3:].split(":", 2) |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
129 key = key.strip().lower() |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
130 |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
131 if key in ['ldflags', 'cflags']: |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
132 for flag in value.split(): |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
133 magics[key] += [flag] |
54
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
134 elif key == "args": |
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
135 # Split arguments respecting quotes |
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
136 for argument in re.findall(r'(?:[^\s,"]|"(?:\\.|[^"])*")+', value): |
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
137 magics['args'] += [argument.strip('"')] |
52
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
138 |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
139 return magics |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
140 |
0 | 141 def do_execute(self, code, silent, store_history=True, |
142 user_expressions=None, allow_stdin=False): | |
52
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
143 |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
144 magics = self._filter_magics(code) |
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
145 |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
146 with self.new_temp_file(suffix='.c') as source_file: |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
147 source_file.write(code) |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
148 source_file.flush() |
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
149 with self.new_temp_file(suffix='.out') as binary_file: |
52
0ef931211f77
Added cflags and ldflags magics
Ben Spoor <ben.spoor@altran.com>
parents:
50
diff
changeset
|
150 p = self.compile_with_gcc(source_file.name, binary_file.name, magics['cflags'], magics['ldflags']) |
33 | 151 while p.poll() is None: |
35 | 152 p.write_contents() |
153 p.write_contents() | |
33 | 154 if p.returncode != 0: # Compilation failed |
34 | 155 self._write_to_stderr( |
156 "[C kernel] GCC exited with code {}, the executable will not be executed".format( | |
157 p.returncode)) | |
33 | 158 return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], |
159 'user_expressions': {}} | |
0 | 160 |
54
7857f8eea835
Add args magic to provide cli-args to user program
Ben Spoor <ben.spoor@altran.com>
parents:
52
diff
changeset
|
161 p = self.create_jupyter_subprocess([self.master_path, binary_file.name] + magics['args']) |
33 | 162 while p.poll() is None: |
35 | 163 p.write_contents() |
164 p.write_contents() | |
33 | 165 |
166 if p.returncode != 0: | |
34 | 167 self._write_to_stderr("[C kernel] Executable exited with code {}".format(p.returncode)) |
2
b46b2e5b6c08
Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents:
0
diff
changeset
|
168 return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} |
7
aa54c85303b6
Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents:
4
diff
changeset
|
169 |
aa54c85303b6
Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents:
4
diff
changeset
|
170 def do_shutdown(self, restart): |
aa54c85303b6
Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents:
4
diff
changeset
|
171 """Cleanup the created source code files and executables when shutting down the kernel""" |
aa54c85303b6
Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents:
4
diff
changeset
|
172 self.cleanup_files() |