Merge branch 'develop' into main

.github/workflows/main.yml

@@ -11,6 +11,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'false'
+      - run: python3 ci/readme_filter.py
       - name: Doxygen Action
         uses: mattnotmitt/[email protected]
         with:

.gitlab-ci.yml

@@ -7,6 +7,7 @@ stages:
   - analyze
   - test
   - leak-check
+  - perf-check
   - docs
   - deploy
 
@@ -284,14 +285,50 @@ leak-check-leg:
 
 #=======================================================================================================================
 
+perf-check-leg:
+  stage: perf-check
+  tags: [linux]
+  image:
+    name: $MACHINES_CONTAINER_REG/leg/builder-full:latest
+    entrypoint: [""]
+  needs: [fetch-ci-scripts,build-leg,install-gamsdist-leg]
+  when: manual
+  script:
+    - SUITE_NAME=gdxfiles
+    - PERF_CALL="gams gdxperf procTreeMemMonitor=1 --RUN=suite --SUITE=${SUITE_NAME} --RUNDEFAULT=0 --RUNCAPI=1"
+    - ARCHIVE_NAME="gdxfiles.zip"
+    - curl https://cloud.gams.com/s/pxSEJNMWszJn3if/download/gdxfiles.zip > $ARCHIVE_NAME
+    - unzip $ARCHIVE_NAME -d $SUITE_NAME
+    - rm $ARCHIVE_NAME
+    - cp -R /cache/gams-installs/`cat gams_folder_leg.txt` mygamsdist # need local copy as we modify libgdxcclib64.so
+    - cp libgdxcclib64.so mygamsdist
+    - GAMS_PATH=`pwd`/mygamsdist/
+    - ${GAMS_PATH}apilib gdxperf
+    - PATH=$GAMS_PATH:$PATH LD_LIBRARY_PATH="$GAMS_PATH" GMSPYTHONLIB="$GAMS_PATH/GMSPython/lib/libpython3.8.so" ${PERF_CALL} | tee cppgdx.log
+    - rm -rf mygamsdist
+    - python3 ci/fetch_gams.py fetch_and_install dist43 $SSH_KEY_PORTING
+    - python3 ci/fetch_gams.py version $PF_CUSTOM_BRANCH $SSH_KEY_PORTING > gams43_folder_leg.txt
+    - GAMS_PATH=/cache/gams-installs/`cat gams43_folder_leg.txt`/
+    - ${GAMS_PATH}apilib gdxperf
+    - PATH=$GAMS_PATH:$PATH LD_LIBRARY_PATH="$GAMS_PATH" GMSPYTHONLIB="$GAMS_PATH/GMSPython/lib/libpython3.8.so" ${PERF_CALL} | tee delphigdx.log
+    - python3 ci/perflogparse.py | tee perf.json
+  artifacts:
+    name: gdxperf-logs
+    paths: [cppgdx.log,delphigdx.log,perf.json]
+    expire_in: 1 day
+
+#=======================================================================================================================
+
 doxygen-html:
   stage: docs
   tags: [linux]
   image:
     name: $MACHINES_CONTAINER_REG/leg/builder-full:latest
     entrypoint: [""]
   script:
+    - python3 ci/readme_filter.py
     - doxygen
+    - rm README_filtered.md
   artifacts:
     name: doxygen-html-docs
     paths: [docs/html]

CMakeLists.txt

@@ -1,5 +1,3 @@
-# NOTE: make sure to copy gclgms.h from products/src/apiwrapper into src/
-
 cmake_minimum_required(VERSION 3.16)
 project(gdx)
 set(CMAKE_CXX_STANDARD 17)
@@ -10,28 +8,36 @@ if(CCACHE_FOUND)
 	set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
 endif(CCACHE_FOUND)
 
-set(mylibs dl pthread)
+set(mylibs dl pthread m)
+
+set(CMAKE_SKIP_RPATH FALSE)
+set(CMAKE_SKIP_BUILD_RPATH FALSE)
+
+if (APPLE)
+	set(CMAKE_BUILD_RPATH @executable_path/.)
+	if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 14)
+		set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-deprecated-non-prototype")
+	endif ()
+else()
+	set(CMAKE_BUILD_RPATH $ORIGIN)
+endif (APPLE)
+
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DGC_NO_MUTEX")
 if (MSVC)
-	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17 /W3 /EHs /D_CRT_SECURE_NO_WARNINGS /wd4267")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17 /W3 /EHs /D_CRT_SECURE_NO_WARNINGS /wd4267 /wd4996")
 	set(mylibs "")
 endif (MSVC)
 
 if (UNIX)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wreturn-type -Wmissing-declarations -Wno-unknown-pragmas")
 endif (UNIX)
 
-if (UNIX AND NOT APPLE)
-	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-truncation=0")
-endif ()
-
 if (APPLE)
 	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DZ_HAVE_UNISTD_H")
+elseif(UNIX)
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-truncation=0")
 endif (APPLE)
 
-# Makefile in devel/products/src/gdxio takes care of acquiring sources for zlib, skip in CMake builds so far
-#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-
 set(common
 		src/gmsstrm.h src/gmsstrm.cpp
 		src/gmsdata.h
@@ -59,20 +65,33 @@ set(tests
 		src/tests/datastoragetests.cpp
 )
 
-add_library(gdxcclib64 SHARED ${common} generated/gdxcclib.cpp)
+# Dynamic library / shared object
+add_library(gdxcclib64 SHARED ${common} generated/gdxcclib.cpp )
+if(UNIX)
+	target_compile_options(gdxcclib64 PRIVATE -fvisibility=hidden)
+endif()
 target_include_directories(gdxcclib64 PRIVATE zlib src generated)
+if (APPLE)
+	set(cclib-link-options "-Bdynamic")
+elseif(UNIX) # Linux
+	set(cclib-link-options "-Bdynamic -Wl,-Bsymbolic")
+else() # Windows
+	set(cclib-link-options "")
+endif()
+target_link_libraries(gdxcclib64 ${mylibs} ${cclib-link-options})
 set_property(TARGET gdxcclib64 PROPERTY POSITION_INDEPENDENT_CODE ON)
 
+# Unit test suite (against statically compiled GDX)
 add_executable(gdxtest ${common} ${tests})
 target_include_directories(gdxtest PRIVATE zlib src generated)
 target_link_libraries(gdxtest ${mylibs})
 
+# Compare against Delphi reference behavior when running against GDX performance library files
 if(UNIX)
 	set(gams43-apifiles-path /home/andre/gamsdist43/apifiles/C/api/)
 else()
 	set(gams43-apifiles-path C:/GAMS/43/apifiles/C/api/)
 endif()
-
 if(EXISTS ${gams43-apifiles-path}gdxcc.c)
 add_executable(delphi-diff-tests ${common}  src/tests/doctestmain.cpp
 		src/tests/delphidifftests.cpp  ${gams43-apifiles-path}gdxcc.c)
@@ -83,6 +102,7 @@ target_link_libraries(delphi-diff-tests ${mylibs})
 	endif()
 endif()
 
+# Unit test suite (against GDX dynamic library)
 add_executable(gdxwraptest src/tests/doctestmain.cpp src/tests/gdxtests.cpp generated/gdxcc.c)
 target_link_libraries(gdxwraptest ${mylibs})
 target_include_directories(gdxwraptest PRIVATE src generated)
@@ -93,10 +113,12 @@ target_compile_options(gdxwraptest PRIVATE -DGXFILE_CPPWRAP -DGC_NO_MUTEX)
 #set_property(TARGET gdxcclib64 PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path})
 #set_property(TARGET gdxtest PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path})
 
+# Standalone GDX example program 1
 add_executable(xp_example1 ${common} src/examples/xp_example1.cpp)
 target_include_directories(xp_example1 PRIVATE zlib src generated)
 target_link_libraries(xp_example1 ${mylibs})
 
+# Standalone GDX example program 2
 add_executable(xp_example2 ${common} src/examples/xp_example2.cpp generated/optcc.c)
 target_include_directories(xp_example2 PRIVATE zlib src generated)
 target_link_libraries(xp_example2 ${mylibs})

Doxyfile

@@ -917,7 +917,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = README.md src
+INPUT                  = README_filtered.md src
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -1131,7 +1131,7 @@ FILTER_SOURCE_PATTERNS =
 # (index.html). This can be useful if you have a project on for instance GitHub
 # and want to reuse the introduction page also for the doxygen output.
 
-USE_MDFILE_AS_MAINPAGE = README.md
+USE_MDFILE_AS_MAINPAGE = README_filtered.md
 
 # The Fortran standard specifies that for fixed formatted Fortran code all
 # characters from position 72 are to be considered as comment. A common

README.md

@@ -1,5 +1,6 @@
 # GAMS Data eXchange (GDX)
 
+<!-- skip doxygen begin -->
 ## Table of Contents
 
 * [GAMS Data eXchange (GDX)](#gams-data-exchange-gdx)
@@ -38,6 +39,7 @@
     * [Conversion issues when moving from GAMS 22.5 to 22.6](#conversion-issues-when-moving-from-gams-225-to-226)
     * [Files in the apifiles directory](#files-in-the-apifiles-directory)
 <!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+<!-- skip doxygen end -->
 
 ## Basic information on GDX file format
 
@@ -594,7 +596,7 @@ The following table organizes the functions by category:
 | Longest symbol UEL  | gdxSymbMaxLength gdxUELMaxLength gdxSymbIndxMaxLength                                                                                                                                         |
 | Acronyms            | gdxAcronymIndex gdxAcronymValue gdxAcronymCount gdxAcronymGetInfo gdxAcronymSetInfo                                                                                                           |
 
-
+<!-- skip doxygen begin -->
 ## Transition diagram
 
 Some GDX operations only make sense after running other routines for preparation beforehand. For example, writing records
@@ -671,6 +673,7 @@ graph TD
     f_map_elem -->|gdxUELRegisterDone| fr_init
     f_str_elem -->|gdxUELRegisterDone| fr_init
 ```
+<!-- skip doxygen end -->
 
 ## Example programs
 
@@ -1569,33 +1572,25 @@ static void WriteData(String s, double V) {
 - support for domain information
 
 Backward compatibility:
-- GAMS and all gdx utilites will write gdx files in the new format
--  GAMS and all gdx utilites can read older gdx formats
--  The gdxcopy utility can convert between different gdx formats
+- GAMS and all gdx utilities will write gdx files in the new format
+- GAMS and all gdx utilities can read older gdx formats
+- The gdxcopy utility can convert between different gdx formats
   (assuming that dimension and namelength is supported)
 
 Libraries:
-- gdxio.dll is still available but the new library is called
-  gdxcclib64.dll (substitute .dll with the extension for your platform)
-- gdxio.dll cannot read the new gdx format
+- `gdxio.dll` is still available but the new library is called `(lib)gdxcclib64.dll` (substitute `.dll` with the extension for your platform, e.g. `.so` or `.dylib`)
+- `gdxio.dll` cannot read the new gdx format
 
 API:
-- Functions in the library that used to return a boolean, now return
-  an integer (zero for false, non-zero for true)
-- Before we can read or write a gdx file, we need to create a valid gdx object. The
-  function gdxCreate will create such an object
-- The functions gdxOpenRead and gdxOpenWrite no longer create the gdx object pointer,
-  they require an object pointer that has been initialized using gdxCreate or similar
-  functions
+- Functions in the library that used to return a boolean, now return  an integer (zero for false, non-zero for true)
+- Before we can read or write a gdx file, we need to create a valid gdx object. The  function `gdxCreate` will create such an object
+- The functions `gdxOpenRead` and `gdxOpenWrite` no longer create the gdx object pointer, they require an object pointer that has been initialized using gdxCreate or similar functions
 
 ## Files in the apifiles directory
 
 The following sections describe the various files included in the apifiles
-directory. All functions will use the gdxcclib library (like gdxcclib.dll on
+directory. All functions will use the gdxcclib library (like `gdxcclib64.dll` on
 Windows). The entry points in the library can be loaded static (by the operating system)
 or dynamic. Dynamic loading provides more control when an entry point is missing
 or the interface has changed. Static loading will cause an exception to be generated
 for example for a missing entry point without much feedback about the error.
-
-For Delphi/Pascal two different interfaces are available; an object interface and a
-function interface.

ci/perflogparse.py

@@ -0,0 +1,38 @@
+import argparse
+import re
+import json
+
+
+def parse(fn):
+    with open(fn) as fp:
+        contents = fp.read()
+    res = dict(memoryMB=dict())
+    m = re.findall(r'(\d+\.\d+)\sMB\)', contents)
+    for ix, mem_type in enumerate(['RSS', 'VSS']):
+        res['memoryMB'][mem_type] = float(m[ix])
+    m = re.search(r'elapsed\s(\d+):(\d+):(\d+\.\d+)', contents)
+    hours, minutes, seconds = (float(m.group(i)) for i in range(1, 4))
+    res['runtimeSecs'] = hours * 3600.0 + minutes * 60.0 + seconds
+    return res
+
+
+def main():
+    ap = argparse.ArgumentParser('perflogparse',
+                                 description='Parse and compare logs from apilib/gdxperf from GAMS 43 and master')
+    ap.add_argument('--delphi_log', type=str,
+                    default='delphigdx.log', help='Log from GAMS 43 gdxperf with procTreeMemMonitor=1')
+    ap.add_argument('--cpp_log', type=str,
+                    default='cppgdx.log', help='Log from GAMS master gdxperf with procTreeMemMonitor=1')
+    args = ap.parse_args()
+    lang_fn = [('Delphi', args.delphi_log), ('C++', args.cpp_log)]
+    languages = [lang for lang, fn in lang_fn]
+    res = {lang: parse(fn) for lang, fn in lang_fn}
+    assert(res['C++']['runtimeSecs'] <= res['Delphi']['runtimeSecs'])
+    slowest_runtime = max(res[lang]["runtimeSecs"] for lang in languages)
+    fastest_runtime = min(res[lang]["runtimeSecs"] for lang in languages)
+    res['slowdown'] = f'{round((slowest_runtime-fastest_runtime)/fastest_runtime*100.0, 2)}%'
+    print(json.dumps(res, sort_keys=True, indent=4))
+
+
+if __name__ == '__main__':
+    main()

ci/readme_filter.py

@@ -0,0 +1,15 @@
+with open('README.md') as fp:
+    lines = fp.readlines()
+out_lines = []
+in_ignore = False
+for line in lines:
+    if line.startswith('<!-- skip doxygen begin -->'):
+        in_ignore = True
+        continue
+    if line.startswith('<!-- skip doxygen end -->'):
+        in_ignore = False
+        continue
+    if not in_ignore:
+        out_lines.append(line)
+with open('README_filtered.md', 'w') as fp:
+    fp.write(''.join(out_lines))