Windows support

Pygolang stopped to work on Windows in 2019 starting from 8fa3c15 (Start using
Cython and providing Cython/nogil API). Restore it now.

Author: navytux

.gitignore

@@ -10,6 +10,8 @@ build/
 *.so.*
 *.dylib
 *.dll
+*.lib
+*.exp
 *.pyd
 *_dsoinfo.py
 

MANIFEST.in

@@ -3,6 +3,7 @@ include golang/libgolang.h
 include golang/runtime/libgolang.cpp
 include golang/runtime/libpyxruntime.cpp
 include golang/pyx/runtime.h
+include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
 include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
 include golang/runtime/internal.h
 include golang/runtime/internal/atomic.h
@@ -33,6 +34,8 @@ include golang/sync_test.cpp
 include golang/time.h
 include golang/time.cpp
 include golang/_testing.h
+include golang/_compat/windows/strings.h
+include golang/_compat/windows/unistd.h
 recursive-include golang    *.py *.pxd *.pyx *.toml *.txt*
 recursive-include gpython   *.py
 recursive-include 3rdparty  *.h

golang/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2022  Nexedi SA and Contributors.
+# Copyright (C) 2018-2023  Nexedi SA and Contributors.
 #                          Kirill Smelkov <[email protected]>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -42,6 +42,9 @@
 import inspect, sys
 import decorator, six
 
+import setuptools_dso
+setuptools_dso.dylink_prepare_dso('golang.runtime.libgolang')
+
 from golang._golang import _pysys_exc_clear as _sys_exc_clear
 
 # @func is a necessary decorator for functions for selected golang features to work.

golang/_compat/windows/strings.h

@@ -0,0 +1,30 @@
+#ifndef _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
+#define _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
+//
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <[email protected]>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+#include <string.h>
+
+static inline void bzero(void *p, size_t n) {
+    memset(p, '\0', n);
+}
+
+#endif  // _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
+

golang/_compat/windows/unistd.h

@@ -0,0 +1,31 @@
+#ifndef _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
+#define _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
+//
+// Copyright (C) 2023  Nexedi SA and Contributors.
+//                     Kirill Smelkov <[email protected]>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// stub unistd.h to be used on Windows where it is absent.
+// we need this because e.g. `cimport posix.stat` forces inclusion of unistd.h
+// even if we use part of posix.stat that is available everywhere.
+
+#include <io.h>
+
+#define O_CLOEXEC  _O_NOINHERIT
+
+#endif  // _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H

golang/fmt.cpp

@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020  Nexedi SA and Contributors.
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <[email protected]>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -48,7 +48,7 @@ string _vsprintf(const char *format, va_list argp) {
     return string(buf.get(), buf.get() + nchar); // without trailing '\0'
 }
 
-string sprintf(const string &format, ...) {
+string sprintf(const string format, ...) {
     va_list argp;
     va_start(argp, format);
     string str = fmt::_vsprintf(format.c_str(), argp);
@@ -64,7 +64,7 @@ string sprintf(const char *format, ...) {
     return str;
 }
 
-error ___errorf(const string &format, ...) {
+error ___errorf(const string format, ...) {
     va_list argp;
     va_start(argp, format);
     error err = errors::New(fmt::_vsprintf(format.c_str(), argp));

golang/fmt.h

@@ -1,7 +1,7 @@
 #ifndef _NXD_LIBGOLANG_FMT_H
 #define _NXD_LIBGOLANG_FMT_H
 
-// Copyright (C) 2019-2020  Nexedi SA and Contributors.
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <[email protected]>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -40,7 +40,7 @@ namespace golang {
 namespace fmt {
 
 // sprintf formats text into string.
-LIBGOLANG_API string sprintf(const string &format, ...);
+LIBGOLANG_API string sprintf(const string format, ...);
 
 
 // intseq<i1, i2, ...> and intrange<n> are used by errorf to handle %w.
@@ -75,7 +75,7 @@ namespace {
 //
 // format suffix ": %w" is additionally handled as in Go with
 // `errorf("... : %w", ..., err)` creating error that can be unwrapped back to err.
-LIBGOLANG_API error ___errorf(const string& format, ...);
+LIBGOLANG_API error ___errorf(const string format, ...);
 LIBGOLANG_API error ___errorfTryWrap(const string& format, error last_err, ...);
 LIBGOLANG_API string ___error_str(error err);
 
@@ -111,7 +111,10 @@ inline error errorf(const string& format, Argv... argv) {
 // `const char *` overloads just to catch format mistakes as
 // __attribute__(format) does not work with std::string.
 LIBGOLANG_API string sprintf(const char *format, ...)
-                                __attribute__ ((format (printf, 1, 2)));
+#ifndef _MSC_VER
+                                __attribute__ ((format (printf, 1, 2)))
+#endif
+	;
 
 // cannot use __attribute__(format) for errorf as we add %w handling.
 // still `const char *` overload is useful for performance.

golang/golang_test.py

@@ -1739,8 +1739,28 @@ def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw):   # -> retcode, st
         pathv.extend(envpath.split(os.pathsep))
     env['PYTHONPATH'] = os.pathsep.join(pathv)
 
+    # set $PYTHONIOENCODING to encoding of stdin/stdout/stderr
+    # we need to do it because on Windows `python x.py | ...` runs with stdio
+    # encoding set to cp125X even if just `python x.py` runs with stdio
+    # encoding=UTF-8.
+    if 'PYTHONIOENCODING' not in env:
+        enc = set([_.encoding for _ in (sys.stdin, sys.stdout, sys.stderr)])
+        if None in enc:         # without -s pytest uses _pytest.capture.DontReadFromInput
+            enc.remove(None)    # with None .encoding
+        assert len(enc) == 1
+        env['PYTHONIOENCODING'] = enc.pop()
+
     p = Popen(argv, stdin=(PIPE if stdin else None), stdout=stdout, stderr=stderr, env=env, **kw)
     stdout, stderr = p.communicate(stdin)
+
+    # on windows print emits \r\n instead of just \n
+    # normalize that to \n in *out
+    if os.name == 'nt':
+        if stdout is not None:
+            stdout = stdout.replace(b'\r\n', b'\n')
+        if stderr is not None:
+            stderr = stderr.replace(b'\r\n', b'\n')
+
     return p.returncode, stdout, stderr
 
 # pyrun runs `sys.executable argv... <stdin`.
@@ -1807,6 +1827,13 @@ def assertDoc(want, got):
     got = got.replace(dir_pygolang,  "PYGOLANG") # /home/x/.../pygolang -> PYGOLANG
     got = got.replace(udir_pygolang, "PYGOLANG") # ~/.../pygolang       -> PYGOLANG
 
+    # got: normalize PYGOLANG\a\b\c -> PYGOLANG/a/b/c
+    #                a\b\c\d.py  -> a/b/c/d.py
+    def _(m):
+        return m.group(0).replace(os.path.sep, '/')
+    got = re.sub(r"(?<=PYGOLANG)[^\s]+(?=\s)",  _, got)
+    got = re.sub(r"([\w\\\.]+)(?=\.py)",        _, got)
+
     # want: process conditionals
     # PY39(...) -> ...   if py ≥ 3.9 else ø  (inline)
     # `... +PY39` -> ... if py ≥ 3.9 else ø  (whole line)
@@ -1862,8 +1889,10 @@ def f(x, y=3, z=4, *argv, **kw): pass
 
 
 # readfile returns content of file @path.
-def readfile(path):
-    with open(path, "r") as f:
+def readfile(path): # -> bytes
+    # on windows in text mode files are opened with encoding=locale.getdefaultlocale()
+    # which is CP125X instead of UTF-8. -> manually decode as 'UTF-8'
+    with open(path, "rb") as f:
         return f.read()
 
 # abbrev_home returns path with user home prefix abbreviated with ~.

golang/libgolang.h

@@ -176,6 +176,12 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#include <fcntl.h>
+#ifdef _MSC_VER // no mode_t on msvc
+typedef int mode_t;
+#endif
+
+
 // DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
 #if defined _WIN32 || defined __CYGWIN__
   #define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
@@ -829,7 +835,7 @@ struct _interface {
 
 protected:
     // don't use destructor -> use decref
-    ~_interface();
+    LIBGOLANG_API ~_interface();
 };
 typedef refptr<_interface> interface;
 

golang/os.cpp

@@ -37,7 +37,8 @@
 // GLIBC <  2.32 provides sys_siglist but not sigdescr_np in its headers
 // cut this short
 // (on darwing sys_siglist declaration is normally provided)
-#ifndef __APPLE__
+// (on windows sys_siglist is not available at all)
+#if !(defined(__APPLE__) || defined(_WIN32))
 extern "C" {
     extern const char * const sys_siglist[];
 }
@@ -248,7 +249,7 @@ tuple<string, error> ReadFile(const string& path) {
 
 tuple<File, File, error> Pipe() {
     int vfd[2], syserr;
-    syserr = sys::Pipe(vfd);
+    syserr = sys::Pipe(vfd, 0);
     if (syserr != 0)
         return make_tuple(nil, nil, fmt::errorf("pipe: %w", sys::NewErrno(syserr)));
 
@@ -286,8 +287,20 @@ string Signal::String() const {
     const Signal& sig = *this;
     const char *sigstr = nil;
 
+#ifdef _WIN32
+    switch (sig.signo) {
+    case SIGABRT:   return "Aborted";
+    case SIGBREAK:  return "Break";
+    case SIGFPE:    return "Floating point exception";
+    case SIGILL:    return "Illegal instruction";
+    case SIGINT:    return "Interrupt";
+    case SIGSEGV:   return "Segmentation fault";
+    case SIGTERM:   return "Terminated";
+    }
+#else
     if (0 <= sig.signo && sig.signo < NSIG)
         sigstr = ::sys_siglist[sig.signo]; // might be nil as well
+#endif
 
     if (sigstr != nil)
         return string(sigstr);

golang/os.h

@@ -1,7 +1,7 @@
 #ifndef _NXD_LIBGOLANG_OS_H
 #define _NXD_LIBGOLANG_OS_H
 //
-// Copyright (C) 2019-2022  Nexedi SA and Contributors.
+// Copyright (C) 2019-2023  Nexedi SA and Contributors.
 //                          Kirill Smelkov <[email protected]>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -61,7 +61,7 @@ class _File : public object {
     ~_File();
     friend File _newFile(_libgolang_ioh* ioh, const string& name);
 public:
-    void decref();
+    LIBGOLANG_API void decref();
 
 public:
     LIBGOLANG_API string  Name()  const;
@@ -95,9 +95,15 @@ class _File : public object {
 
 // Open opens file @path.
 LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
-        mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR |
+        mode_t mode =
+#if !defined(_MSC_VER)
+                      S_IRUSR | S_IWUSR | S_IXUSR |
                       S_IRGRP | S_IWGRP | S_IXGRP |
-                      S_IROTH | S_IWOTH | S_IXOTH);
+                      S_IROTH | S_IWOTH | S_IXOTH
+#else
+                      _S_IREAD | _S_IWRITE
+#endif
+        );
 
 // NewFile wraps OS-level file-descriptor into File.
 // The ownership of sysfd is transferred to File.