Mercurial > hg > Members > aka > jupyter_CbC_kernel

annotate jupyter_c_kernel/kernel.py @ 38:cb587f21c8bb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Rename Jupytersubprocess and comment it
author Brendan Rius <brendan@omixy.com>
date Sat, 30 Apr 2016 23:05:43 +0100
parents aac47bc07111
children fecdf8733f3b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
1 from queue import Queue, Empty
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
2 from threading import Thread
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
3
0
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
4 from ipykernel.kernelbase import Kernel
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
5 import subprocess
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
6 import tempfile
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
7 import os
36
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
8 import os.path as path
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
9
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
10
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
11 class RealTimeSubprocess(subprocess.Popen):
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
12 """
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
13 A subprocess that allows to read its stdout and stderr in real time
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
14 """
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
15
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
16 def __init__(self, cmd, write_to_stdout, write_to_stderr):
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
17 """
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
18 :param cmd: the command to execute
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
19 :param write_to_stdout: a callable that will be called with chunks of data from stdout
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
20 :param write_to_stderr: a callable that will be called with chunks of data from stderr
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
21 """
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
22 self._write_to_stdout = write_to_stdout
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
23 self._write_to_stderr = write_to_stderr
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
24
37
aac47bc07111 Fix subprocess bufsize & master compilation
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 36
diff changeset
25 super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
26
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
27 self._stdout_queue = Queue()
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
28 self._stdout_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stdout, self._stdout_queue))
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
29 self._stdout_thread.daemon = True
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
30 self._stdout_thread.start()
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
31
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
32 self._stderr_queue = Queue()
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
33 self._stderr_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stderr, self._stderr_queue))
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
34 self._stderr_thread.daemon = True
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
35 self._stderr_thread.start()
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
36
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
37 @staticmethod
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
38 def _enqueue_output(stream, queue):
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
39 """
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
40 Add chunks of data from a stream to a queue until the stream is empty.
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
41 """
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
42 for line in iter(lambda: stream.read(4096), b''):
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
43 queue.put(line)
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
44 stream.close()
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
45
35
e05900c7967e Remove logging
Brendan Rius <brendan@omixy.com>
parents: 34
diff changeset
46 def write_contents(self):
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
47 """
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
48 Write the available content from stdin and stderr where specifid you the instance was created
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
49 :return:
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
50 """
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
51 try:
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
52 stdout_contents = self._stdout_queue.get_nowait()
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
53 except Empty:
35
e05900c7967e Remove logging
Brendan Rius <brendan@omixy.com>
parents: 34
diff changeset
54 pass
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
55 else:
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
56 self._write_to_stdout(stdout_contents)
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
57 try:
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
58 stderr_contents = self._stderr_queue.get_nowait()
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
59 except Empty:
35
e05900c7967e Remove logging
Brendan Rius <brendan@omixy.com>
parents: 34
diff changeset
60 pass
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
61 else:
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
62 self._write_to_stderr(stderr_contents)
0
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
63
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
64
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
65 class CKernel(Kernel):
20
696cc0775abb Change package name
Brendan Rius <brendan@omixy.com>
parents: 9
diff changeset
66 implementation = 'jupyter_c_kernel'
0
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
67 implementation_version = '1.0'
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
68 language = 'c'
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
69 language_version = 'C11'
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
70 language_info = {'name': 'c',
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
71 'mimetype': 'text/plain',
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
72 'file_extension': 'c'}
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
73 banner = "C kernel.\n" \
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
74 "Uses gcc, compiles in C11, and creates source code files and executables in temporary folder.\n"
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
75
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
76 def __init__(self, *args, **kwargs):
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
77 super(CKernel, self).__init__(*args, **kwargs)
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
78 self.files = []
36
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
79 mastertemp = tempfile.mkstemp(suffix='.out')
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
80 os.close(mastertemp[0])
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
81 self.master_path = mastertemp[1]
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
82 filepath = path.join(path.dirname(path.realpath(__file__)), '..', 'resources', 'master.c')
37
aac47bc07111 Fix subprocess bufsize & master compilation
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 36
diff changeset
83 subprocess.call(['gcc', filepath, '-std=c11', '-rdynamic', '-ldl', '-o', self.master_path])
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
84
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
85 def cleanup_files(self):
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
86 """Remove all the temporary files created by the kernel"""
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
87 for file in self.files:
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
88 os.remove(file)
36
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
89 os.remove(self.master_path)
0
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
90
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
91 def new_temp_file(self, **kwargs):
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
92 """Create a new temp file to be deleted when the kernel shuts down"""
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
93 # We don't want the file to be deleted when closed, but only when the kernel stops
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
94 kwargs['delete'] = False
9
101e99452042 Dockerized project :)
Brendan Rius <brendan@omixy.com>
parents: 7
diff changeset
95 kwargs['mode'] = 'w'
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
96 file = tempfile.NamedTemporaryFile(**kwargs)
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
97 self.files.append(file.name)
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
98 return file
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
99
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
100 def _write_to_stdout(self, contents):
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
101 self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': contents})
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
102
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
103 def _write_to_stderr(self, contents):
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
104 self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': contents})
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
105
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
106 def create_jupyter_subprocess(self, cmd):
38
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
107 return RealTimeSubprocess(cmd,
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
108 lambda contents: self._write_to_stdout(contents.decode()),
cb587f21c8bb Rename Jupytersubprocess and comment it
Brendan Rius <brendan@omixy.com>
parents: 37
diff changeset
109 lambda contents: self._write_to_stderr(contents.decode()))
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
110
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
111 def compile_with_gcc(self, source_filename, binary_filename):
36
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
112 args = ['gcc', source_filename, '-std=c11', '-fPIC', '-shared', '-rdynamic', '-o', binary_filename]
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
113 return self.create_jupyter_subprocess(args)
0
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
114
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
115 def do_execute(self, code, silent, store_history=True,
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
116 user_expressions=None, allow_stdin=False):
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
117 with self.new_temp_file(suffix='.c') as source_file:
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
118 source_file.write(code)
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
119 source_file.flush()
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
120 with self.new_temp_file(suffix='.out') as binary_file:
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
121 p = self.compile_with_gcc(source_file.name, binary_file.name)
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
122 while p.poll() is None:
35
e05900c7967e Remove logging
Brendan Rius <brendan@omixy.com>
parents: 34
diff changeset
123 p.write_contents()
e05900c7967e Remove logging
Brendan Rius <brendan@omixy.com>
parents: 34
diff changeset
124 p.write_contents()
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
125 if p.returncode != 0: # Compilation failed
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
126 self._write_to_stderr(
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
127 "[C kernel] GCC exited with code {}, the executable will not be executed".format(
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
128 p.returncode))
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
129 return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [],
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
130 'user_expressions': {}}
0
50ea00cf5896 Initial commit
Brendan Rius <brendan@omixy.com>
parents:
diff changeset
131
36
8acbab5a9f21 Remove stream buffering on stdout & stderr
Louis 'Kureuil' Person <louis.person@epitech.eu>
parents: 35
diff changeset
132 p = self.create_jupyter_subprocess([self.master_path, binary_file.name])
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
133 while p.poll() is None:
35
e05900c7967e Remove logging
Brendan Rius <brendan@omixy.com>
parents: 34
diff changeset
134 p.write_contents()
e05900c7967e Remove logging
Brendan Rius <brendan@omixy.com>
parents: 34
diff changeset
135 p.write_contents()
33
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
136
Brendan Rius <brendan@omixy.com>
parents: 29
diff changeset
137 if p.returncode != 0:
34
Brendan Rius <brendan@omixy.com>
parents: 33
diff changeset
138 self._write_to_stderr("[C kernel] Executable exited with code {}".format(p.returncode))
2
b46b2e5b6c08 Improve error management by separating stdout and stderr
Brendan Rius <brendan@omixy.com>
parents: 0
diff changeset
139 return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}}
7
aa54c85303b6 Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents: 4
diff changeset
140
aa54c85303b6 Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents: 4
diff changeset
141 def do_shutdown(self, restart):
aa54c85303b6 Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents: 4
diff changeset
142 """Cleanup the created source code files and executables when shutting down the kernel"""
aa54c85303b6 Remove temporary files when shutting down the kernel
Brendan Rius <brendan@omixy.com>
parents: 4
diff changeset
143 self.cleanup_files()