Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

python a_compile.py on an Apple M1 mac results in error #1

Open
ahrib opened this issue Sep 29, 2022 · 2 comments
Open

python a_compile.py on an Apple M1 mac results in error #1

ahrib opened this issue Sep 29, 2022 · 2 comments

Comments

@ahrib
Copy link

ahrib commented Sep 29, 2022

Attempting to run python a_compile.py on an Apple M1 mac results in error.

If it helps, building with cmake works as expected when using a similar project https://github.com/neurolabusc/pigz-bench

environment details

  System Version: macOS 12.6 (21G115)
  Kernel Version: Darwin 21.6.0
  Boot Volume: Macintosh HD
  Boot Mode: Normal
  Secure Virtual Memory: Enabled
  System Integrity Protection: Enabled
  Model Name: MacBook Pro
  Model Identifier: MacBookPro18,3
  Chip: Apple M1 Pro
  Total Number of Cores: 10 (8 performance and 2 efficiency)
  Memory: 16 GB
  System Firmware Version: 7459.141.1
  OS Loader Version: 7459.141.1

full console output below

  CMake Deprecation Warning at /Users/andrej/Documents/Projects/pigzbench-master/zlib-madler/CMakeLists.txt:1 (cmake_minimum_required):
   Compatibility with CMake < 2.8.12 will be removed from a future version of
   CMake.
 
   Update the VERSION argument <min> value or use a ...<max> suffix to tell
   CMake that the project does not need compatibility with older versions.
 
 
 -- Configuring done
 CMake Warning (dev):
   Policy CMP0042 is not set: MACOSX_RPATH is enabled by default.  Run "cmake
   --help-policy CMP0042" for policy details.  Use the cmake_policy command to
   set the policy and suppress this warning.
 
   MACOSX_RPATH is not specified for the following targets:
 
    zlib
 
 This warning is for project developers.  Use -Wno-dev to suppress it.
 
 -- Generating done
 -- Build files have been written to: /Users/andrej/Documents/Projects/pigzbench-master/pigz-madler/build
 Consolidate compiler generated dependencies of target zlibstatic
 [ 80%] Built target zlibstatic
 Consolidate compiler generated dependencies of target pigz
 [ 85%] Linking C executable pigz
 [100%] Built target pigz
 /Users/andrej/Documents/Projects/pigzbench-master/pigz-madler/build/pigz->/Users/andrej/Documents/Projects/pigzbench-master/exe/pigz-madler
 -- Configuring done
 -- Generating done
 -- Build files have been written to: /Users/andrej/Documents/Projects/pigzbench-master/pigz-cloudflare/build
 Consolidate compiler generated dependencies of target zlib
 [ 81%] Built target zlib
 Consolidate compiler generated dependencies of target pigz
 [ 86%] Linking C executable pigz
 Undefined symbols for architecture arm64:
   "_get_crc_table", referenced from:
       _main in pigz.c.o
 ld: symbol(s) not found for architecture arm64
 clang: error: linker command failed with exit code 1 (use -v to see invocation)
 make[2]: *** [pigz] Error 1
 make[1]: *** [CMakeFiles/pigz.dir/all] Error 2
 make: *** [all] Error 2
 Traceback (most recent call last):
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 815, in move
     os.rename(src, real_dst)
 FileNotFoundError: [Errno 2] No such file or directory: '/Users/andrej/Documents/Projects/pigzbench-master/pigz-cloudflare/build/Release/pigz' -> '/Users/andrej/Documents/Projects/pigzbench-master/exe/pigz-cloudflare'
 
 During handling of the above exception, another exception occurred:
 
 Traceback (most recent call last):
   File "/Users/andrej/Documents/Projects/pigzbench-master/a_compile.py", line 174, in <module>
     compile_pigz(args.rebuild)
   File "/Users/andrej/Documents/Projects/pigzbench-master/a_compile.py", line 164, in compile_pigz
     shutil.move(pigzexe, outnm)
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 835, in move
     copy_function(src, real_dst)
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 434, in copy2
     copyfile(src, dst, follow_symlinks=follow_symlinks)
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 254, in copyfile
     with open(src, 'rb') as fsrc:
 FileNotFoundError: [Errno 2] No such file or directory: '/Users/andrej/Documents/Projects/pigzbench-master/pigz-cloudflare/build/Release/pigz'
@nmoinvaz
Copy link
Member

zlib may need to target a particular branch or commit. I don't have an M1 so that is all I can help.

@neurolabusc
Copy link

@nmoinvaz this version of a_compile.py works for my M2. It uses the cmake from @ningfei. I do think it only builds the develop branch of zlib-ng, but it does build the system (Madler), CloudFlare and zlib-ng variations.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import argparse
import stat
import shutil
import subprocess
import platform
import zipfile
from distutils.dir_util import copy_tree

def rmtree(top):
    """Delete folder and contents: shutil.rmtree has issues with read-only files on Windows"""

    for (root, dirs, files) in os.walk(top, topdown=False):
        for name in files:
            filename = os.path.join(root, name)
            os.chmod(filename, stat.S_IWUSR)
            os.remove(filename)
        for name in dirs:
            os.rmdir(os.path.join(root, name))
    os.rmdir(top)


def install_silesia_corpus():
    """Install popular Silesia corpus"""

    basedir = os.getcwd()
    corpusdir = os.path.join(basedir, 'silesia')
    if os.path.isdir(corpusdir):
        return
    try:
        os.mkdir(corpusdir)
    except OSError:
        print('Creation of the directory {} failed' .format(corpusdir))
    cmd = 'git clone https://github.com/MiloszKrajewski/SilesiaCorpus silesia'
    print("Installing "+corpusdir)
    subprocess.call(cmd, shell=True)
    os.chdir(corpusdir)
    fnm = 'README.md'
    if os.path.isfile(fnm):
        os.remove(fnm)
    ext = '.zip'
    for item in os.listdir(corpusdir):  # loop through items in dir
        print("+"+item)
        if item.endswith(ext):  # check for ".zip" extension
            file_name = os.path.abspath(item)  # get full path of files
            print(file_name)
            zip_ref = zipfile.ZipFile(file_name)  # create zipfile object
            zip_ref.extractall(corpusdir)  # extract file to dir
            zip_ref.close()  # close file
            os.remove(file_name)  # delete zipped file
    os.chdir(basedir)


def install_neuro_corpus():
    """Install neuroimaging corpus"""

    basedir = os.getcwd()
    corpusdir = os.path.join(basedir, 'corpus')
    if os.path.isdir(corpusdir):
        return
    try:
        os.mkdir(corpusdir)
    except OSError:
        print('Creation of the directory {} failed' .format(exedir))
    cmd = 'git clone https://github.com/neurolabusc/zlib-bench.git'
    subprocess.call(cmd, shell=True)
    indir = os.path.join(basedir, 'zlib-bench', 'corpus')
    tmpfile = os.path.join(indir, 'README')
    if os.path.isfile(tmpfile):
        os.remove(tmpfile)
    copy_tree(indir, corpusdir)
    indir = os.path.join(basedir, 'zlib-bench')
    rmtree(indir)


def compile_pigz(rebuild=True):
    """compile variants of pigz"""

    methods = ['Cloudflare', 'ng','System']
    basedir = os.getcwd()
    exedir = os.path.join(basedir, 'exe')
    if os.path.isdir(exedir):
        rmtree(exedir)
    os.mkdir(exedir)
    repodir = os.path.join(basedir, 'pigz')
    if os.path.isdir(repodir):
        rmtree(repodir)
    ext = ''
    if platform.system() == 'Windows':
        ext = '.exe'
    cmd = 'git clone https://github.com/ningfei/pigz.git'
    subprocess.call(cmd, shell=True)
    if os.path.isdir(exedir):
        rmtree(exedir)
    os.mkdir(exedir)
    builddir = os.path.join(repodir, 'build')
    pigzexe = os.path.join(builddir, 'bin', 'pigz') + ext
    for method in methods:
        os.chdir(repodir)
        if os.path.isdir(builddir):
          rmtree(builddir)
        os.mkdir(builddir)
        os.chdir(builddir)
        cmd = 'cmake -DZLIB_IMPLEMENTATION=' + method + ' ..'
        subprocess.call(cmd, shell=True)
        cmd = 'make'
        if platform.system() == 'Windows':
            cmd = 'cmake --build . --config Release'
        subprocess.call(cmd, shell=True)
        outnm = os.path.join(exedir, 'pigz' + method + ext)
        print (pigzexe + '->' + outnm)
        shutil.move(pigzexe, outnm)


if __name__ == '__main__':
    """compile variants of pigz and sample compression corpus"""

    install_neuro_corpus()
    install_silesia_corpus()
    compile_pigz()

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

3 participants