From 7ece5f868f0ca8975956f77a0fc097f13df3bbf8 Mon Sep 17 00:00:00 2001 From: Keith Rutkowski Date: Wed, 24 Mar 2021 12:27:13 -0400 Subject: [PATCH] Initial code commit --- .github/FUNDING.yml | 1 + .github/workflows/CI.yml | 84 +++++++++++++++++ .github/workflows/TagBot.yml | 14 +++ .gitignore | 5 ++ LICENSE | 21 +++++ Project.toml | 11 +++ README.md | 96 ++++++++++++++++++++ pkgs/alsa.jl | 14 +++ pkgs/libc.jl | 170 +++++++++++++++++++++++++++++++++++ pkgs/linux.jl | 15 ++++ pkgs/rdma.jl | 39 ++++++++ pkgs/sys/Project.toml | 7 ++ pkgs/sys/src/sys.jl | 108 ++++++++++++++++++++++ src/System.jl | 36 ++++++++ test/Project.toml | 2 + test/runtests.jl | 34 +++++++ 16 files changed, 657 insertions(+) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/TagBot.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Project.toml create mode 100644 README.md create mode 100644 pkgs/alsa.jl create mode 100644 pkgs/libc.jl create mode 100644 pkgs/linux.jl create mode 100644 pkgs/rdma.jl create mode 100644 pkgs/sys/Project.toml create mode 100644 pkgs/sys/src/sys.jl create mode 100644 src/System.jl create mode 100644 test/Project.toml create mode 100644 test/runtests.jl diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..147accc --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: analytech-solutions diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..a4f44db --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,84 @@ +name: CI +on: + - push + - pull_request +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.container }} + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + strategy: + fail-fast: false + matrix: + version: + - '1.5' + - '1.6-nightly' + - 'nightly' + os: + - ubuntu-latest + container: + # - 'alpine:3' + - 'amazonlinux:2' + - 'archlinux:base' + - 'centos:7' + - 'centos:8' + - 'debian:8' + - 'debian:9' + - 'debian:10' + - 'fedora:34' + # - 'opensuse/leap:15' + - 'ubuntu:16.04' + - 'ubuntu:18.04' + - 'ubuntu:20.04' + arch: + - x64 + # include: + # - os: macos-latest + # version: '1.5' + # arch: x64 + # - os: macos-latest + # version: '1.6-nightly' + # arch: x64 + # - os: macos-latest + # version: 'nightly' + # arch: x64 + # - os: windows-latest + # version: '1.5' + # arch: x64 + # - os: windows-latest + # version: '1.6-nightly' + # arch: x64 + # - os: windows-latest + # version: 'nightly' + # arch: x64 + steps: + - name: Install deps + if: ${{ runner.os == 'Linux' }} + run: | + set -eu + + case "${{ matrix.container }}" in + 'debian:'* | 'ubuntu:'*) + apt-get update + apt-get install -y --no-install-recommends curl ca-certificates libc6-dev libasound2 libasound2-dev + ;; + 'amazonlinux:'* | 'centos:'* | 'fedora:'*) + yum -y install tar gzip glibc-headers alsa-lib alsa-lib-devel + ;; + 'opensuse/leap:'*) + zypper install -y tar gzip curl + ;; + 'archlinux:'*) + yes | pacman -Syu --needed alsa-lib + ;; + *) + ;; + esac + + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-runtest@latest diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml new file mode 100644 index 0000000..4b10a57 --- /dev/null +++ b/.github/workflows/TagBot.yml @@ -0,0 +1,14 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fd27895 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.jl.cov +*.jl.*.cov +*.jl.mem +deps/deps.jl +/gen diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..003bb19 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Analytech Solutions + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..53a8bb8 --- /dev/null +++ b/Project.toml @@ -0,0 +1,11 @@ +name = "System" +uuid = "75a07032-1394-4a60-8011-bf81691479e2" +authors = ["Keith Rutkowski "] +version = "0.1.0" + +[deps] +CBinding = "d43a6710-96b8-4a2d-833c-c424785e5374" + +[compat] +julia = "^1.5" +CBinding = "^1.0.2" diff --git a/README.md b/README.md new file mode 100644 index 0000000..fc837f7 --- /dev/null +++ b/README.md @@ -0,0 +1,96 @@ +# System.jl + +[![Build Status](https://github.com/analytech-solutions/System.jl/workflows/CI/badge.svg)](https://github.com/analytech-solutions/System.jl/actions) + +A framework for interfacing with system-installed software from Julia. + +System.jl allows for the use of trusted system software without relying on the binaries downloaded as Julia artifacts. +We view System.jl as an essential component for proprietary and secure computing environments. +This package does not yet support all platforms (only common Linux distributions at present), but it provides a path to that goal. +It also requires that the header files are installed for libraries so that bindings can be automatically generated from them. + + +# Usage + +System.jl is a framework providing bindings to operating system and system-installed software API's. +System resources are available as Julia packages that encapsulate dynamically generated bindings (automatically created by [CBinding.jl](https://github.com/analytech-solutions/CBinding.jl) when you import the package). +These packages can be found in the `System.jl/pkgs` directory and are only available for use once System.jl has been imported. +Therefore, [similar to Revise.jl](https://timholy.github.io/Revise.jl/stable/config), `using System` must occur before any packages utilizing the framework are loaded, or just add it to your `~/.julia/config/startup.jl` file. + +Bindings for a system resource are loaded with the `@sys using libxyz` macro syntax. +The bindings can always be referenced with the CBinding.jl `c"..."` string macro, but usually the bindings are free of name collisions so Julian names are available as well. + +```jl +julia> using System + +julia> @sys using libc.C99 + +julia> c"printf"("printf is the best!\n") +printf is the best! +20 + +julia> @sys using alsa.libasound + +julia> for val in 0:Int(SND_PCM_STREAM_LAST) + name = snd_pcm_stream_name(val) + c"printf"("%s\n", name) + end +PLAYBACK +CAPTURE +``` + + +# Developing a framework package + +Packages within the System.jl framework, found in `System.jl/pkgs`, are not known about by Pkg.jl when packages are installed. +Therefore, the framework packages are unable to use _any_ packages that are not referenced by the System.jl package itself (its dependencies are all Pkg.jl knows about). +Framework packages are generally light-weight uses of CBinding.jl, but the special `sys` package introduces tools to facilitate the process. + +It provides the `@pkgconf` macro to automatically inject the dependency packages' compilation command line arguments and header file inclusions in order to prepare both the Julia and C definitions needed to declare the package's bindings. +The following example demonstrates the usage of this macro: + +```jl +module libpkg + using sys + + @pkgconf begin + using libdep1, libdep2 + c`-I/path/to/include -L/path/to/libs -lpkg` + c""" + #include + #include + """ji + end +end +``` + +And what the manually written equivalent might look like: + +```jl +module libpkg + using sys + + using libdep1 + using libdep2 + + c`-L/dep1/lib -ldep1 -DDEP2_USE_DEP1=1 -L/dep2/lib -ldep2 -I/path/to/include -L/path/to/lib -lpkg` + + c""" + #include + #include + """s + + c""" + #include + #include + """s + + c""" + #include + #include + """ji +end +``` + +Further details will become available as the package grows and is tested on more systems. + diff --git a/pkgs/alsa.jl b/pkgs/alsa.jl new file mode 100644 index 0000000..8f4a35e --- /dev/null +++ b/pkgs/alsa.jl @@ -0,0 +1,14 @@ +module alsa + module libasound + using sys + + @pkgconf begin + using libc.POSIX + + c`-lasound` + c""" + #include + """ji + end + end +end diff --git a/pkgs/libc.jl b/pkgs/libc.jl new file mode 100644 index 0000000..850f2e2 --- /dev/null +++ b/pkgs/libc.jl @@ -0,0 +1,170 @@ +module libc + const HEADERS = (; + C89 = ( + "assert.h", + "ctype.h", + "errno.h", + "float.h", + "limits.h", + "locale.h", + "math.h", + "setjmp.h", + "signal.h", + "stdarg.h", + "stddef.h", + "stdio.h", + "stdlib.h", + "string.h", + "time.h", + ), + + C95 = ( + :C89, + "iso646.h", + "wchar.h", + "wctype.h", + ), + + C99 = ( + :C95, + "fenv.h", + "inttypes.h", + "stdbool.h", + "stdint.h", + "tgmath.h", + ), + + C11 = ( + :C99, + "stdalign.h", + "stdatomic.h", + "stdnoreturn.h", + "threads.h", + "uchar.h", + ), + + POSIX = ( + :C99, + "aio.h", + "arpa/inet.h", + "complex.h", + "cpio.h", + "dirent.h", + "dlfcn.h", + "fcntl.h", + "fmtmsg.h", + "fnmatch.h", + "ftw.h", + "glob.h", + "grp.h", + "iconv.h", + "langinfo.h", + "libgen.h", + "monetary.h", + "mqueue.h", + # "ndbm.h", + "net/if.h", + "netdb.h", + "netinet/in.h", + "netinet/tcp.h", + "nl_types.h", + "poll.h", + "pthread.h", + "pwd.h", + "regex.h", + "sched.h", + "search.h", + "semaphore.h", + "spawn.h", + "strings.h", + # "stropts.h", + "sys/ipc.h", + "sys/mman.h", + "sys/msg.h", + "sys/resource.h", + "sys/select.h", + "sys/sem.h", + "sys/shm.h", + "sys/socket.h", + "sys/stat.h", + "sys/statvfs.h", + "sys/time.h", + "sys/times.h", + "sys/types.h", + "sys/uio.h", + "sys/un.h", + "sys/utsname.h", + "sys/wait.h", + "syslog.h", + "tar.h", + "termios.h", + # "trace.h", + "ulimit.h", + "unistd.h", + "utime.h", + "utmpx.h", + "wordexp.h", + ), + ) + + const ALIASES = (; + C90 = :C89, + C17 = :C11, + + GNU89 = :C89, + GNU90 = :C90, + GNU95 = :C95, + GNU99 = :C99, + GNU11 = :C11, + GNU17 = :C17, + + C = :GNU17, + ) + + + @eval export $(keys(HEADERS)...), $(keys(ALIASES)...) + for std in keys(HEADERS) + hdrs = HEADERS[std] + hasmore = true + while hasmore + hasmore = false + for ind in eachindex(hdrs) + if hdrs[ind] isa Symbol + hdrs = (hdrs[1:ind-1]..., HEADERS[hdrs[ind]]..., hdrs[ind+1:end]...) + hasmore = true + break + end + end + end + + ctx = :(c``) + if std === :POSIX + ctx.args[3] = "-std=c99 -D_REENTRANT -D_POSIX_SOURCE -D_POSIX_C_SOURCE=200809L" + else + ctx.args[3] = "-std=$(lowercase(String(std)))" + end + + bindings = :(c""Jcu) + bindings.args[3] = join(map(hdr -> "#include <$(hdr)>", hdrs), '\n') + + try + @eval module $(std) + using sys + + @pkgconf begin + $(ctx) + $(bindings) + end + end + catch + std === :C11 || error("Headers for $(std) Standard Library not found") + end + end + + for std in keys(ALIASES) + try + @eval const $(std) = $(ALIASES[std]) + catch + end + end +end diff --git a/pkgs/linux.jl b/pkgs/linux.jl new file mode 100644 index 0000000..6277250 --- /dev/null +++ b/pkgs/linux.jl @@ -0,0 +1,15 @@ +module linux + using sys + + @pkgconf begin + using libc.POSIX + + c`` + c""" + #include + #include + #include + // add other linux/*.h here... + """ji + end +end diff --git a/pkgs/rdma.jl b/pkgs/rdma.jl new file mode 100644 index 0000000..2cf7361 --- /dev/null +++ b/pkgs/rdma.jl @@ -0,0 +1,39 @@ +module rdma + export libibverbs, librdmacm + + + module libibverbs + using sys + + @pkgconf begin + using libc.POSIX, linux + + c`-libverbs` + c""" + #include + """jiw + end + + function __init__() + ENV["IBV_FORK_SAFE"] = "1" + end + end + + + module librdmacm + using sys + + @pkgconf begin + using libc.POSIX, linux, ..libibverbs + + c`-lrdmacm` + c""" + #include + """ji + end + + function __init__() + ENV["RDMAV_FORK_SAFE"] = "1" + end + end +end diff --git a/pkgs/sys/Project.toml b/pkgs/sys/Project.toml new file mode 100644 index 0000000..406819f --- /dev/null +++ b/pkgs/sys/Project.toml @@ -0,0 +1,7 @@ +name = "sys" +uuid = "7022cf9a-419e-427c-a992-ed81cd5f5d41" +authors = ["Keith Rutkowski "] +version = "0.1.0" + +[deps] +System = "75a07032-1394-4a60-8011-bf81691479e2" diff --git a/pkgs/sys/src/sys.jl b/pkgs/sys/src/sys.jl new file mode 100644 index 0000000..26ffd5f --- /dev/null +++ b/pkgs/sys/src/sys.jl @@ -0,0 +1,108 @@ +module sys + using System + using System.CBinding + + + export @sys, @c_cmd, @c_str, @pkgconf, @pkgconf_cmd + + + macro pkgconf(exprs...) return pkgconf(__module__, __source__, exprs...) end + + function pkgconf(mod::Module, loc::LineNumberNode, expr::Expr) + exprs = Base.is_expr(expr, :block) ? expr.args : [expr] + + deps = [] + usings = [] + cmds = [] + strs = [] + extras = [] + for expr in exprs + # TODO: handle predicated statements + + if Base.is_expr(expr, :using) + for e in expr.args + if Base.is_expr(e, :(:)) && length(e.args) >= 1 + e = e.args[1] + end + + ind = 1 + while e.args[ind] == :. + ind += 1 + end + + dep = nothing + for ind in ind:length(e.args) + dep = isnothing(dep) ? e.args[ind] : :($(dep).$(e.args[ind])) + push!(usings, Expr(:import, Expr(:., e.args[1:ind]...))) + end + push!(deps, dep) + end + push!(usings, expr) + elseif Base.is_expr(expr, :macrocall) && expr.args[1] == Symbol("@c_cmd") + push!(cmds, expr) + elseif Base.is_expr(expr, :macrocall) && expr.args[1] == Symbol("@pkgconf_cmd") + push!(cmds, @eval @macroexpand1 $(expr)) + elseif Base.is_expr(expr, :macrocall) && expr.args[1] == Symbol("@c_str") + push!(strs, expr) + else + push!(extras, esc(expr)) + end + end + + + flags = map(cmds) do cmd + expr = :(``) + expr.args[3] = cmd.args[3] + return :($(esc(expr)).exec...) + end + + prereqs = [] + ctx = :(``) + ctx.args[1] = :($(CBinding).var"@c_cmd") + for dep in deps + push!(prereqs, :($(Expr(:$, :($(esc(dep))._DEFS...))))) + ctx.args[3] *= " \$($(dep)._FLAGS)" + end + ctx.args[3] *= " \$(_FLAGS)" + + defs = map(strs) do str + str = copy(str) + length(str.args) < 4 && push!(str.args, "") + str.args[4] = "s" + return QuoteNode(str) + end + + deps = map(esc, deps) + + return quote + $(usings...) + $(extras...) + + const $(esc(:_FLAGS)) = ($(flags...),) + $(esc(ctx)) + + @eval begin + $(prereqs...) + $(strs...) + end + const $(esc(:_DEFS)) = ($(defs...),) + end + end + + + macro pkgconf_cmd(exprs...) return pkgconf_cmd(__module__, __source__, exprs...) end + + function pkgconf_cmd(mod::Module, loc::LineNumberNode, str::String, opts::String = "") + opt = [] + 'f' in opts && push!(opt, "--cflags") + 'l' in opts && push!(opt, "--libs") + + # TODO: handle spaces in paths, etc... + flags = String(read(`pkg-config $(opt) $(str)`)) + + expr = :(``) + expr.args[1] = :($(CBinding).var"@c_cmd") + expr.args[3] = flags + return esc(expr) + end +end diff --git a/src/System.jl b/src/System.jl new file mode 100644 index 0000000..87fd1e3 --- /dev/null +++ b/src/System.jl @@ -0,0 +1,36 @@ +module System + using CBinding + + + export @sys + export @c_str, isqualifiedwith, unqualifiedtype, bitstype + + + macro sys(exprs...) return sys(__module__, __source__, exprs...) end + + function sys(mod::Module, loc::LineNumberNode, expr::Expr) + Base.is_expr(expr, :using, 1) || Base.is_expr(expr, :import, 1) || error("Expected a `using ...` or `import ...` statement in @sys usage") + + path = expr.args[1] + if Base.is_expr(path, :(:)) && length(path.args) >= 1 + path = path.args[1] + end + name = path.args[1] + + sym = gensym(name) + pushfirst!(path.args, :Base, :Main, sym) + + return quote + @eval Base.Main module $(sym) + import $(name) + end + $(expr) + end + end + + + function __init__() + push!(Base.LOAD_PATH, joinpath(dirname(@__DIR__), "pkgs")) + end +end + diff --git a/test/Project.toml b/test/Project.toml new file mode 100644 index 0000000..0c36332 --- /dev/null +++ b/test/Project.toml @@ -0,0 +1,2 @@ +[deps] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..005b055 --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,34 @@ +using Test +using System + + +# NOTE: tests require certain headers and libraries to be installed +# reference `System.jl/.github/workflows/CI.yml` for commands + + +@testset "System.jl" begin + @test @eval begin + @sys using sys + true + end + + + @test @eval begin + @sys using libc + true + end + + + if Sys.islinux() + @test @eval begin + @sys using linux + true + end + + @test @eval begin + @sys using alsa + true + end + end +end +