# HG changeset patch # User musou_aka <> # Date 1529737913 -32400 # Node ID 70e6b10d92202650eca866d9cc41d6e900632393 # Parent a36609a3f8b61cae7945a7368ddcca7eb5eda83d rename CbC kernel diff -r a36609a3f8b6 -r 70e6b10d9220 Dockerfile --- a/Dockerfile Sun Jun 03 22:21:48 2018 +0200 +++ b/Dockerfile Sat Jun 23 16:11:53 2018 +0900 @@ -5,10 +5,10 @@ WORKDIR /tmp -COPY ./ jupyter_c_kernel/ +COPY ./ jupyter_CbC_kernel/ -RUN pip install --no-cache-dir jupyter_c_kernel/ -RUN cd jupyter_c_kernel && install_c_kernel --user +RUN pip install --no-cache-dir jupyter_CbC_kernel/ +RUN cd jupyter_CbC_kernel && install_c_kernel --user WORKDIR /home/$NB_USER/ diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_CbC_kernel/__init__.py diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_CbC_kernel/__main__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/jupyter_CbC_kernel/__main__.py Sat Jun 23 16:11:53 2018 +0900 @@ -0,0 +1,3 @@ +from ipykernel.kernelapp import IPKernelApp +from .kernel import CbCKernel +IPKernelApp.launch_instance(kernel_class=CbCKernel) diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_CbC_kernel/install_c_kernel --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/jupyter_CbC_kernel/install_c_kernel Sat Jun 23 16:11:53 2018 +0900 @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +import json +import os +import sys +import argparse + +from jupyter_client.kernelspec import KernelSpecManager +from IPython.utils.tempdir import TemporaryDirectory + +kernel_json = { + "argv": [ + "python3", + "-m", + "jupyter_CbC_kernel", + "-f", + "{connection_file}" + ], + "display_name": "CbC", + "language": "CbC" +} + + +def install_my_kernel_spec(user=True, prefix=None): + with TemporaryDirectory() as td: + os.chmod(td, 0o755) # Starts off as 700, not user readable + with open(os.path.join(td, 'kernel.json'), 'w') as f: + json.dump(kernel_json, f, sort_keys=True) + # TODO: Copy resources once they're specified + + print('Installing IPython kernel spec') + KernelSpecManager().install_kernel_spec(td, 'CbC', user=user, replace=True, prefix=prefix) + + +def _is_root(): + try: + return os.geteuid() == 0 + except AttributeError: + return False # assume not an admin on non-Unix platforms + + +def main(argv=[]): + parser = argparse.ArgumentParser( + description='Install KernelSpec for C Kernel' + ) + prefix_locations = parser.add_mutually_exclusive_group() + + prefix_locations.add_argument( + '--user', + help='Install KernelSpec in user homedirectory', + action='store_false' if _is_root() else 'store_true' + ) + prefix_locations.add_argument( + '--sys-prefix', + help='Install KernelSpec in sys.prefix. Useful in conda / virtualenv', + action='store_true', + dest='sys_prefix' + ) + prefix_locations.add_argument( + '--prefix', + help='Install KernelSpec in this prefix', + default=None + ) + + args = parser.parse_args() + + if args.sys_prefix: + prefix = sys.prefix + user = None + elif args.user: + prefix = None + user = True + else: + prefix = args.prefix + user = None + + install_my_kernel_spec(user=user, prefix=prefix) + + +if __name__ == '__main__': + main(argv=sys.argv) diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_CbC_kernel/kernel.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/jupyter_CbC_kernel/kernel.py Sat Jun 23 16:11:53 2018 +0900 @@ -0,0 +1,172 @@ +from queue import Queue +from threading import Thread + +from ipykernel.kernelbase import Kernel +import re +import subprocess +import tempfile +import os +import os.path as path + + +class RealTimeSubprocess(subprocess.Popen): + """ + A subprocess that allows to read its stdout and stderr in real time + """ + + def __init__(self, cmd, write_to_stdout, write_to_stderr): + """ + :param cmd: the command to execute + :param write_to_stdout: a callable that will be called with chunks of data from stdout + :param write_to_stderr: a callable that will be called with chunks of data from stderr + """ + self._write_to_stdout = write_to_stdout + self._write_to_stderr = write_to_stderr + + super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) + + self._stdout_queue = Queue() + self._stdout_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stdout, self._stdout_queue)) + self._stdout_thread.daemon = True + self._stdout_thread.start() + + self._stderr_queue = Queue() + self._stderr_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stderr, self._stderr_queue)) + self._stderr_thread.daemon = True + self._stderr_thread.start() + + @staticmethod + def _enqueue_output(stream, queue): + """ + Add chunks of data from a stream to a queue until the stream is empty. + """ + for line in iter(lambda: stream.read(4096), b''): + queue.put(line) + stream.close() + + def write_contents(self): + """ + Write the available content from stdin and stderr where specified when the instance was created + :return: + """ + + def read_all_from_queue(queue): + res = b'' + size = queue.qsize() + while size != 0: + res += queue.get_nowait() + size -= 1 + return res + + stdout_contents = read_all_from_queue(self._stdout_queue) + if stdout_contents: + self._write_to_stdout(stdout_contents) + stderr_contents = read_all_from_queue(self._stderr_queue) + if stderr_contents: + self._write_to_stderr(stderr_contents) + + +class CbCKernel(Kernel): + implementation = 'jupyter_CbC_kernel' + implementation_version = '1.0' + language = 'CbC' + language_version = 'C11' + language_info = {'name': 'CbC', + 'mimetype': 'text/plain', + 'file_extension': '.c'} + banner = "CbC kernel.\n" \ + "Uses gcc, compiles in C11, and creates source code files and executables in temporary folder.\n" + + def __init__(self, *args, **kwargs): + super(CbCKernel, self).__init__(*args, **kwargs) + self.files = [] + mastertemp = tempfile.mkstemp(suffix='.out') + os.close(mastertemp[0]) + self.master_path = mastertemp[1] + filepath = path.join(path.dirname(path.realpath(__file__)), 'resources', 'master.c') + subprocess.call(['gcc', filepath, '-std=c11', '-rdynamic', '-ldl', '-o', self.master_path]) + + def cleanup_files(self): + """Remove all the temporary files created by the kernel""" + for file in self.files: + os.remove(file) + os.remove(self.master_path) + + def new_temp_file(self, **kwargs): + """Create a new temp file to be deleted when the kernel shuts down""" + # We don't want the file to be deleted when closed, but only when the kernel stops + kwargs['delete'] = False + kwargs['mode'] = 'w' + file = tempfile.NamedTemporaryFile(**kwargs) + self.files.append(file.name) + return file + + def _write_to_stdout(self, contents): + self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': contents}) + + def _write_to_stderr(self, contents): + self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': contents}) + + def create_jupyter_subprocess(self, cmd): + return RealTimeSubprocess(cmd, + lambda contents: self._write_to_stdout(contents.decode()), + lambda contents: self._write_to_stderr(contents.decode())) + + def compile_with_gcc(self, source_filename, binary_filename, cflags=None, ldflags=None): + cflags = ['-std=c11', '-fPIC', '-shared', '-rdynamic'] + cflags + args = ['gcc', source_filename] + cflags + ['-o', binary_filename] + ldflags + return self.create_jupyter_subprocess(args) + + def _filter_magics(self, code): + + magics = {'cflags': [], + 'ldflags': [], + 'args': []} + + for line in code.splitlines(): + if line.startswith('//%'): + key, value = line[3:].split(":", 2) + key = key.strip().lower() + + if key in ['ldflags', 'cflags']: + for flag in value.split(): + magics[key] += [flag] + elif key == "args": + # Split arguments respecting quotes + for argument in re.findall(r'(?:[^\s,"]|"(?:\\.|[^"])*")+', value): + magics['args'] += [argument.strip('"')] + + return magics + + def do_execute(self, code, silent, store_history=True, + user_expressions=None, allow_stdin=False): + + magics = self._filter_magics(code) + + with self.new_temp_file(suffix='.c') as source_file: + source_file.write(code) + source_file.flush() + with self.new_temp_file(suffix='.out') as binary_file: + p = self.compile_with_gcc(source_file.name, binary_file.name, magics['cflags'], magics['ldflags']) + while p.poll() is None: + p.write_contents() + p.write_contents() + if p.returncode != 0: # Compilation failed + self._write_to_stderr( + "[CbC kernel] GCC exited with code {}, the executable will not be executed".format( + p.returncode)) + return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], + 'user_expressions': {}} + + p = self.create_jupyter_subprocess([self.master_path, binary_file.name] + magics['args']) + while p.poll() is None: + p.write_contents() + p.write_contents() + + if p.returncode != 0: + self._write_to_stderr("[CbC kernel] Executable exited with code {}".format(p.returncode)) + return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} + + def do_shutdown(self, restart): + """Cleanup the created source code files and executables when shutting down the kernel""" + self.cleanup_files() diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_CbC_kernel/resources/master.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/jupyter_CbC_kernel/resources/master.c Sat Jun 23 16:11:53 2018 +0900 @@ -0,0 +1,31 @@ +#include +#include +#include + +typedef int (*main_t)(int, char **, char **); + +int main(int argc, char **argv, char **envp) +{ + char *error = NULL; + + setbuf(stdout, NULL); + setbuf(stderr, NULL); + if (argc < 2) { + fprintf(stderr, "USAGE: %s PROGRAM\nWhere PROGRAM is the user's program to supervise\n", argv[0]); + return EXIT_FAILURE; + } + void *userhandle = dlopen(argv[1], RTLD_LAZY); + if (userhandle == NULL) { + fprintf(stderr, "%s: %s\n", argv[0], dlerror()); + return EXIT_FAILURE; + } + dlerror(); + main_t usermain = dlsym(userhandle, "main"); + if ((error = dlerror()) != NULL) { + fprintf(stderr, "%s: %s\n", argv[0], error); + return EXIT_FAILURE; + } + + /* Call Users main, but make master.c invisible by removing first argument */ + return usermain(argc-1, argv+1, envp); +} diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_c_kernel/__init__.py diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_c_kernel/__main__.py --- a/jupyter_c_kernel/__main__.py Sun Jun 03 22:21:48 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,3 +0,0 @@ -from ipykernel.kernelapp import IPKernelApp -from .kernel import CKernel -IPKernelApp.launch_instance(kernel_class=CKernel) diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_c_kernel/install_c_kernel --- a/jupyter_c_kernel/install_c_kernel Sun Jun 03 22:21:48 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import json -import os -import sys -import argparse - -from jupyter_client.kernelspec import KernelSpecManager -from IPython.utils.tempdir import TemporaryDirectory - -kernel_json = { - "argv": [ - "python3", - "-m", - "jupyter_c_kernel", - "-f", - "{connection_file}" - ], - "display_name": "C", - "language": "c" -} - - -def install_my_kernel_spec(user=True, prefix=None): - with TemporaryDirectory() as td: - os.chmod(td, 0o755) # Starts off as 700, not user readable - with open(os.path.join(td, 'kernel.json'), 'w') as f: - json.dump(kernel_json, f, sort_keys=True) - # TODO: Copy resources once they're specified - - print('Installing IPython kernel spec') - KernelSpecManager().install_kernel_spec(td, 'c', user=user, replace=True, prefix=prefix) - - -def _is_root(): - try: - return os.geteuid() == 0 - except AttributeError: - return False # assume not an admin on non-Unix platforms - - -def main(argv=[]): - parser = argparse.ArgumentParser( - description='Install KernelSpec for C Kernel' - ) - prefix_locations = parser.add_mutually_exclusive_group() - - prefix_locations.add_argument( - '--user', - help='Install KernelSpec in user homedirectory', - action='store_false' if _is_root() else 'store_true' - ) - prefix_locations.add_argument( - '--sys-prefix', - help='Install KernelSpec in sys.prefix. Useful in conda / virtualenv', - action='store_true', - dest='sys_prefix' - ) - prefix_locations.add_argument( - '--prefix', - help='Install KernelSpec in this prefix', - default=None - ) - - args = parser.parse_args() - - if args.sys_prefix: - prefix = sys.prefix - user = None - elif args.user: - prefix = None - user = True - else: - prefix = args.prefix - user = None - - install_my_kernel_spec(user=user, prefix=prefix) - - -if __name__ == '__main__': - main(argv=sys.argv) diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_c_kernel/kernel.py --- a/jupyter_c_kernel/kernel.py Sun Jun 03 22:21:48 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,172 +0,0 @@ -from queue import Queue -from threading import Thread - -from ipykernel.kernelbase import Kernel -import re -import subprocess -import tempfile -import os -import os.path as path - - -class RealTimeSubprocess(subprocess.Popen): - """ - A subprocess that allows to read its stdout and stderr in real time - """ - - def __init__(self, cmd, write_to_stdout, write_to_stderr): - """ - :param cmd: the command to execute - :param write_to_stdout: a callable that will be called with chunks of data from stdout - :param write_to_stderr: a callable that will be called with chunks of data from stderr - """ - self._write_to_stdout = write_to_stdout - self._write_to_stderr = write_to_stderr - - super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) - - self._stdout_queue = Queue() - self._stdout_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stdout, self._stdout_queue)) - self._stdout_thread.daemon = True - self._stdout_thread.start() - - self._stderr_queue = Queue() - self._stderr_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stderr, self._stderr_queue)) - self._stderr_thread.daemon = True - self._stderr_thread.start() - - @staticmethod - def _enqueue_output(stream, queue): - """ - Add chunks of data from a stream to a queue until the stream is empty. - """ - for line in iter(lambda: stream.read(4096), b''): - queue.put(line) - stream.close() - - def write_contents(self): - """ - Write the available content from stdin and stderr where specified when the instance was created - :return: - """ - - def read_all_from_queue(queue): - res = b'' - size = queue.qsize() - while size != 0: - res += queue.get_nowait() - size -= 1 - return res - - stdout_contents = read_all_from_queue(self._stdout_queue) - if stdout_contents: - self._write_to_stdout(stdout_contents) - stderr_contents = read_all_from_queue(self._stderr_queue) - if stderr_contents: - self._write_to_stderr(stderr_contents) - - -class CKernel(Kernel): - implementation = 'jupyter_c_kernel' - implementation_version = '1.0' - language = 'c' - language_version = 'C11' - language_info = {'name': 'c', - 'mimetype': 'text/plain', - 'file_extension': '.c'} - banner = "C kernel.\n" \ - "Uses gcc, compiles in C11, and creates source code files and executables in temporary folder.\n" - - def __init__(self, *args, **kwargs): - super(CKernel, self).__init__(*args, **kwargs) - self.files = [] - mastertemp = tempfile.mkstemp(suffix='.out') - os.close(mastertemp[0]) - self.master_path = mastertemp[1] - filepath = path.join(path.dirname(path.realpath(__file__)), 'resources', 'master.c') - subprocess.call(['gcc', filepath, '-std=c11', '-rdynamic', '-ldl', '-o', self.master_path]) - - def cleanup_files(self): - """Remove all the temporary files created by the kernel""" - for file in self.files: - os.remove(file) - os.remove(self.master_path) - - def new_temp_file(self, **kwargs): - """Create a new temp file to be deleted when the kernel shuts down""" - # We don't want the file to be deleted when closed, but only when the kernel stops - kwargs['delete'] = False - kwargs['mode'] = 'w' - file = tempfile.NamedTemporaryFile(**kwargs) - self.files.append(file.name) - return file - - def _write_to_stdout(self, contents): - self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': contents}) - - def _write_to_stderr(self, contents): - self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': contents}) - - def create_jupyter_subprocess(self, cmd): - return RealTimeSubprocess(cmd, - lambda contents: self._write_to_stdout(contents.decode()), - lambda contents: self._write_to_stderr(contents.decode())) - - def compile_with_gcc(self, source_filename, binary_filename, cflags=None, ldflags=None): - cflags = ['-std=c11', '-fPIC', '-shared', '-rdynamic'] + cflags - args = ['gcc', source_filename] + cflags + ['-o', binary_filename] + ldflags - return self.create_jupyter_subprocess(args) - - def _filter_magics(self, code): - - magics = {'cflags': [], - 'ldflags': [], - 'args': []} - - for line in code.splitlines(): - if line.startswith('//%'): - key, value = line[3:].split(":", 2) - key = key.strip().lower() - - if key in ['ldflags', 'cflags']: - for flag in value.split(): - magics[key] += [flag] - elif key == "args": - # Split arguments respecting quotes - for argument in re.findall(r'(?:[^\s,"]|"(?:\\.|[^"])*")+', value): - magics['args'] += [argument.strip('"')] - - return magics - - def do_execute(self, code, silent, store_history=True, - user_expressions=None, allow_stdin=False): - - magics = self._filter_magics(code) - - with self.new_temp_file(suffix='.c') as source_file: - source_file.write(code) - source_file.flush() - with self.new_temp_file(suffix='.out') as binary_file: - p = self.compile_with_gcc(source_file.name, binary_file.name, magics['cflags'], magics['ldflags']) - while p.poll() is None: - p.write_contents() - p.write_contents() - if p.returncode != 0: # Compilation failed - self._write_to_stderr( - "[C kernel] GCC exited with code {}, the executable will not be executed".format( - p.returncode)) - return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], - 'user_expressions': {}} - - p = self.create_jupyter_subprocess([self.master_path, binary_file.name] + magics['args']) - while p.poll() is None: - p.write_contents() - p.write_contents() - - if p.returncode != 0: - self._write_to_stderr("[C kernel] Executable exited with code {}".format(p.returncode)) - return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} - - def do_shutdown(self, restart): - """Cleanup the created source code files and executables when shutting down the kernel""" - self.cleanup_files() diff -r a36609a3f8b6 -r 70e6b10d9220 jupyter_c_kernel/resources/master.c --- a/jupyter_c_kernel/resources/master.c Sun Jun 03 22:21:48 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -#include -#include -#include - -typedef int (*main_t)(int, char **, char **); - -int main(int argc, char **argv, char **envp) -{ - char *error = NULL; - - setbuf(stdout, NULL); - setbuf(stderr, NULL); - if (argc < 2) { - fprintf(stderr, "USAGE: %s PROGRAM\nWhere PROGRAM is the user's program to supervise\n", argv[0]); - return EXIT_FAILURE; - } - void *userhandle = dlopen(argv[1], RTLD_LAZY); - if (userhandle == NULL) { - fprintf(stderr, "%s: %s\n", argv[0], dlerror()); - return EXIT_FAILURE; - } - dlerror(); - main_t usermain = dlsym(userhandle, "main"); - if ((error = dlerror()) != NULL) { - fprintf(stderr, "%s: %s\n", argv[0], error); - return EXIT_FAILURE; - } - - /* Call Users main, but make master.c invisible by removing first argument */ - return usermain(argc-1, argv+1, envp); -}