Skip to content

Commit 0de88f2

Browse files
Add python magic
1 parent 0ff8b19 commit 0de88f2

File tree

6 files changed

+318
-27
lines changed

6 files changed

+318
-27
lines changed

CMakeLists.txt

Lines changed: 34 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ option(XEUS_CPP_BUILD_STATIC "Build xeus-cpp static library" ON)
8787
OPTION(XEUS_CPP_BUILD_SHARED "Split xcpp build into executable and library" ON)
8888
OPTION(XEUS_CPP_BUILD_EXECUTABLE "Build the xcpp executable" ON)
8989

90-
OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp with the xeus shared library (instead of the static library)" ON)
91-
OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp with the xeus shared library (instead of the static library)" ON)
90+
OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp with the xeus shared library (instead of the static library)" ON)
91+
OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp with the xeus shared library (instead of the static library)" ON)
9292

9393
OPTION(XEUS_CPP_EMSCRIPTEN_WASM_BUILD "Build for wasm with emscripten" OFF)
9494

@@ -129,15 +129,18 @@ else()
129129
add_compile_options(-fexceptions)
130130
endif ()
131131

132-
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Intel")
132+
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR
133+
CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR
134+
CMAKE_CXX_COMPILER_ID MATCHES "Intel")
135+
133136
if(NOT XEUS_CPP_EMSCRIPTEN_WASM_BUILD)
134137
add_compile_options(-Wunused-parameter -Wextra -Wreorder)
135138
endif()
136139

137-
138140
CHECK_CXX_COMPILER_FLAG("-std=c++17" HAS_CPP_17_FLAG)
139141
if (HAS_CPP_17_FLAG)
140-
add_compile_options(-std=c++17)
142+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
143+
set(CMAKE_CXX_STANDARD 17)
141144
else ()
142145
message(FATAL_ERROR "Unsupported compiler -- xeus requires C++17 support!")
143146
endif ()
@@ -151,8 +154,8 @@ endif()
151154
find_package(CppInterOp REQUIRED CONFIG PATHS "${CPPINTEROP_DIR}" "${CPPINTEROP_DIR}/lib")
152155
find_package(argparse REQUIRED)
153156
find_package(pugixml REQUIRED)
154-
##set(Python_FIND_VIRTUALENV ONLY)
155-
##find_package(Python COMPONENTS Interpreter Development)
157+
set(Python_FIND_VIRTUALENV ONLY)
158+
find_package(Python COMPONENTS Interpreter Development)
156159

157160
# Source files
158161
# ============
@@ -168,6 +171,7 @@ set(XEUS_CPP_SRC
168171
src/xinterpreter.cpp
169172
src/xoptions.cpp
170173
src/xparser.cpp
174+
src/xmagics/pythonexec.cpp
171175
)
172176

173177
set(XEUS_CPP_MAIN_SRC
@@ -327,23 +331,30 @@ if (XEUS_CPP_BUILD_EXECUTABLE)
327331
xeus_cpp_set_common_options(xcpp)
328332
xeus_cpp_set_kernel_options(xcpp)
329333
target_link_libraries(xcpp PRIVATE xeus-zmq)
334+
set_target_properties(xcpp PROPERTIES
335+
ENABLE_EXPORTS 1
336+
CXX_STANDARD ${CMAKE_CXX_STANDARD})
337+
target_link_libraries(xcpp PUBLIC xeus-cpp pthread Python::Python)
338+
339+
##TODO: We may be need sse RPATH
340+
set_target_properties(xcpp clangCppInterOp PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
341+
if(APPLE)
342+
target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup)
343+
elseif(NOT MSVC)
344+
target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files)
345+
endif()
330346

331-
##set_target_properties(xcpp PROPERTIES
332-
## ENABLE_EXPORTS 1
333-
## CXX_STANDARD ${CMAKE_CXX_STANDARD}
334-
##)
335-
##target_link_libraries(xcpp PUBLIC xeus-cpp pthread Python::Python)
336-
##
337-
###TODO: We may be need sse RPATH
338-
###set_target_properties(xcpp clangCppInterOp PROPERTIES
339-
### INSTALL_RPATH_USE_LINK_PATH TRUE
340-
###)
341-
##if(APPLE)
342-
## target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup)
343-
##elseif(NOT MSVC)
344-
## target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files)
345-
##endif()
346-
347+
target_include_directories(xeus-cpp PUBLIC ${Python_INCLUDE_DIRS})
348+
target_link_libraries(xeus-cpp PUBLIC ${PYTHON_LIBRARIES})
349+
target_link_libraries(xeus-cpp ${PYTHON_LIBRARIES_Development_Main})
350+
set_target_properties(xeus-cpp PROPERTIES
351+
PUBLIC_HEADER "${XEUS_CPP_HEADERS}"
352+
COMPILE_DEFINITIONS "XEUS_CPP_EXPORTS"
353+
PREFIX ""
354+
VERSION ${${PROJECT_NAME}_VERSION}
355+
SOVERSION ${XEUS_CPP_VERSION_MAJOR}
356+
OUTPUT_NAME "libxeus-cpp"
357+
CXX_STANDARD ${CMAKE_CXX_STANDARD})
347358
endif()
348359

349360
if(XEUS_CPP_EMSCRIPTEN_WASM_BUILD)

Dockerfile

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# https://hub.docker.com/r/jupyter/base-notebook/tags
55
ARG BASE_CONTAINER=jupyter/base-notebook
66
ARG BASE_TAG=latest
7-
ARG BUILD_TYPE=Release
7+
ARG BUILD_TYPE=Debug
88

99
FROM $BASE_CONTAINER:$BASE_TAG
1010

@@ -80,7 +80,7 @@ USER ${NB_UID}
8080
ENV NB_PYTHON_PREFIX=${CONDA_DIR} \
8181
KERNEL_PYTHON_PREFIX=${CONDA_DIR} \
8282
# CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/${NB_USER}/include:/home/runner/work/xeus-clang-repl/xeus-clang-repl/clang-dev/clang/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include"
83-
CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/${NB_USER}/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include"
83+
CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include"
8484

8585
WORKDIR "${HOME}"
8686

@@ -131,6 +131,7 @@ RUN \
131131
cppzmq \
132132
xtl \
133133
'clangdev>=17' \
134+
'llvm-openmp' \
134135
pugixml \
135136
cpp-argparse \
136137
zlib \
@@ -173,7 +174,8 @@ RUN \
173174
# Build CppInterOp
174175
#
175176
sys_incs=$(LC_ALL=C c++ -xc++ -E -v /dev/null 2>&1 | LC_ALL=C sed -ne '/starts here/,/End of/p' | LC_ALL=C sed '/^ /!d' | cut -c2- | tr '\n' ':') && \
176-
export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clange:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \
177+
#/usr/include/x86_64-linux-gnu:/usr/include:
178+
export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clang:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \
177179
git clone https://github.com/compiler-research/CppInterOp.git && \
178180
export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \
179181
export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" && \

src/xinterpreter.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,18 @@
2323
#include "xeus-cpp/xbuffer.hpp"
2424
#include "xeus-cpp/xeus_cpp_config.hpp"
2525

26+
#include "xeus-cpp/xinterpreter.hpp"
27+
#include "xeus-cpp/xmagics.hpp"
28+
29+
#include "xinput.hpp"
30+
// #include "xinspect.hpp"
31+
// #include "xmagics/executable.hpp"
32+
// #include "xmagics/execution.hpp"
33+
#include "xmagics/os.hpp"
34+
#include "xmagics/pythonexec.hpp"
35+
#include "xparser.hpp"
36+
#include "xsystem.hpp"
37+
2638
using Args = std::vector<const char*>;
2739

2840
void* createInterpreter(const Args &ExtraArgs = {}) {
@@ -329,6 +341,7 @@ namespace xcpp
329341
// preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("executable", executable(m_interpreter));
330342
// preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("file", writefile());
331343
// preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("timeit", timeit(&m_interpreter));
344+
preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("python", pythonexec());
332345
}
333346

334347
std::string interpreter::get_stdopt(int argc, const char* const* argv)

src/xmagics/os.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
#include "os.hpp"
1616
#include "../xparser.hpp"
1717

18-
#include "xeus-cpp/xoptions.hpp"
18+
//#include "xeus-cpp/xoptions.hpp"
1919

2020
namespace xcpp
2121
{

src/xmagics/pythonexec.cpp

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
//===------------ pythonexec.hpp - Python/C++ Interoperability ------------===//
2+
//
3+
// Licensed under the Apache License v2.0.
4+
// SPDX-License-Identifier: Apache-2.0
5+
//
6+
// The full license is in the file LICENSE, distributed with this software.
7+
//
8+
//===----------------------------------------------------------------------===//
9+
//
10+
// This file defines the Python/C++ interoperability in which two cells within
11+
// the same notebook can be in a either language.
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
#include "Python.h"
16+
17+
#include "pythonexec.hpp"
18+
#include "../xparser.hpp"
19+
20+
#include <cstddef>
21+
#include <fstream>
22+
#include <iostream>
23+
#include <sstream>
24+
#include <string>
25+
#include <vector>
26+
27+
namespace xcpp {
28+
static PyObject *gMainDict = 0;
29+
30+
void pythonexec::startup() {
31+
static bool isInitialized = false;
32+
if (isInitialized)
33+
return;
34+
35+
Py_Initialize();
36+
37+
PyRun_SimpleString("import sys\nprint(sys.path)");
38+
39+
// Import cppyy module
40+
PyObject* cppyyModule = PyImport_ImportModule("cppyy");
41+
if (!cppyyModule) {
42+
PyErr_Print();
43+
Py_Finalize();
44+
return; // Handle import error as needed
45+
}
46+
47+
PyObject* mainModule = PyImport_AddModule("__main__");
48+
PyObject_SetAttrString(mainModule, "cppyy", cppyyModule);
49+
Py_XDECREF(cppyyModule);
50+
isInitialized = true;
51+
// gMainDict = PyModule_GetDict(mainModule);
52+
// Py_INCREF(gMainDict);
53+
// if (!gMainDict)
54+
// printf("Could not add module __main__");
55+
56+
57+
// // Retrieve the dictionary of cppyy module
58+
// PyObject* cppyyDict = PyModule_GetDict(cppyyModule);
59+
// Py_DECREF(cppyyModule);
60+
// if (!cppyyDict) {
61+
// PyErr_Print();
62+
// Py_Finalize();
63+
// return; // Handle retrieval error as needed
64+
// }
65+
66+
// // Add cppyyDict to gMainDict (if needed for further usage)
67+
// PyDict_Update(gMainDict, cppyyDict);
68+
69+
// Py_DECREF(cppyyDict);
70+
// PyRun_SimpleString("import cppyy");
71+
}
72+
73+
argparser pythonexec::get_options() {
74+
argparser argpars{"python", "Start executing Python Cell"};
75+
return argpars;
76+
}
77+
78+
void pythonexec::execute(std::string &line, std::string &cell) {
79+
// std::istringstream iss(line);
80+
// std::vector<std::string> results((std::istream_iterator<std::string>(iss)),
81+
// std::istream_iterator<std::string>());
82+
startup();
83+
84+
// auto argpars = get_options();
85+
// argpars.parse(line);
86+
87+
std::string code;
88+
89+
code += cell;
90+
if (trim(code).empty())
91+
return;
92+
// PyRun_SimpleString(
93+
// "globals_copy_lists = "
94+
// "globals().copy()\nfirst_dict_ints={k:globals_copy_lists[k] for k in "
95+
// "set(globals_copy_lists) if type(globals_copy_lists[k]) == "
96+
// "int}\nfirst_dict_lists={k:globals_copy_lists[k] for k in "
97+
// "set(globals_copy_lists) if type(globals_copy_lists[k]) == list}");
98+
99+
// PyRun_SimpleString("tmp = globals().copy()\nvars = [f'int {k} = {v};' for
100+
// k,v in tmp.items() if type(v) == int and not k.startswith('_') and k!='tmp'
101+
// and k!='In' and k!='Out' and k!='sys' and not hasattr(v,
102+
// '__call__')]\nprint(vars)"); PyRun_SimpleString("b =
103+
// globals().copy()\nnew_ints = ' '.join([f'int {k} = {b[k]};' for k in set(b)
104+
// - set(first_dict) if type(b[k]) == int])\nprint('new_ints: ', new_ints)");
105+
106+
Cpp::BeginStdStreamCapture(Cpp::kStdErr);
107+
Cpp::BeginStdStreamCapture(Cpp::kStdOut);
108+
109+
PyRun_SimpleString(code.c_str());
110+
111+
std::cout << Cpp::EndStdStreamCapture();
112+
std::cerr << Cpp::EndStdStreamCapture();
113+
114+
// PyObject* objectsRepresentation = PyObject_Repr(gMainDict);
115+
// const char* s = PyUnicode_AsUTF8(objectsRepresentation);
116+
// printf("REPR of global dict: %s\n", s);
117+
}
118+
119+
void pythonexec::update_python_dict_var(const char *name, int value) {
120+
if (!gMainDict)
121+
startup();
122+
PyObject *s;
123+
s = PyLong_FromLong(value);
124+
PyDict_SetItemString(gMainDict, name, s);
125+
Py_DECREF(s);
126+
}
127+
128+
void pythonexec::update_python_dict_var_vector(const char *name,
129+
std::vector<int> &data) {
130+
if (!gMainDict)
131+
startup();
132+
PyObject *listObj = PyList_New(data.size());
133+
if (!listObj)
134+
throw std::logic_error("Unable to allocate memory for Python list");
135+
for (unsigned int i = 0; i < data.size(); i++) {
136+
PyObject *num = PyLong_FromLong((int)data[i]);
137+
if (!num) {
138+
Py_DECREF(listObj);
139+
throw std::logic_error("Unable to allocate memory for Python list");
140+
}
141+
PyList_SET_ITEM(listObj, i, num);
142+
}
143+
PyDict_SetItemString(gMainDict, name, listObj);
144+
}
145+
146+
// check python globals
147+
void pythonexec::check_python_globals() {
148+
if (!Py_IsInitialized())
149+
return;
150+
// execute the command
151+
PyRun_SimpleString("print(globals())");
152+
}
153+
154+
// execute a python comand
155+
void pythonexec::exec_python_simple_command(const std::string code) {
156+
if (!Py_IsInitialized())
157+
return;
158+
// execute the command
159+
PyRun_SimpleString(code.c_str());
160+
}
161+
162+
std::string pythonexec::transfer_python_ints_utility() {
163+
if (!Py_IsInitialized())
164+
return " ";
165+
// transfer ints utility
166+
PyRun_SimpleString(
167+
"def getNewInts():\n glob_ints_utils = globals().copy()\n new_ints "
168+
"= ' '.join([f'int {k} = {glob_ints_utils[k]};' for k in "
169+
"set(glob_ints_utils) - set(first_dict_ints) if type(glob_ints_utils[k]) "
170+
"== int])\n return new_ints");
171+
PyObject *ints_result =
172+
PyObject_CallFunction(PyDict_GetItemString(gMainDict, "getNewInts"), 0);
173+
if (!ints_result) {
174+
printf("Could not retrieve Python integers!\n");
175+
return " ";
176+
} else {
177+
std::string newPythonInts = PyUnicode_AsUTF8(ints_result);
178+
// printf("new ints %s\n", PyUnicode_AsUTF8(ints_result));
179+
Py_DECREF(ints_result);
180+
return newPythonInts;
181+
}
182+
}
183+
184+
std::string pythonexec::transfer_python_lists_utility() {
185+
if (!Py_IsInitialized())
186+
return " ";
187+
// transfer lists utility
188+
PyRun_SimpleString("def getNewLists():\n l = globals().copy()\n "
189+
"new_lists = ' '.join([f'int {k}() = {l[k]};' for k in "
190+
"set(l) - set(first_dict_lists) if type(l[k]) == "
191+
"list]).replace('[','{').replace(']','}').replace('(','[')"
192+
".replace(')',']')\n return new_lists");
193+
PyObject *lists_result =
194+
PyObject_CallFunction(PyDict_GetItemString(gMainDict, "getNewLists"), 0);
195+
if (!lists_result) {
196+
printf("Could not retrieve Python lists!\n");
197+
return " ";
198+
} else {
199+
std::string newPythonLists = PyUnicode_AsUTF8(lists_result);
200+
Py_DECREF(lists_result);
201+
return newPythonLists;
202+
}
203+
}
204+
205+
bool pythonexec::python_check_for_initialisation() {
206+
if (!Py_IsInitialized())
207+
return false;
208+
return true;
209+
}
210+
} // namespace xcpp

0 commit comments

Comments
 (0)