From 1d983e9f934bf6aeb9333c763fe1a603b8d8e5c4 Mon Sep 17 00:00:00 2001 From: Patrick Spek Date: Sun, 22 Mar 2020 11:48:23 +0100 Subject: Initial commit --- README.md | 107 +++++++++++++++++++++++++++++++++++++++ bin/rstar | 15 ++++++ etc/dist_moarvm.txt | 2 + etc/dist_nqp.txt | 2 + etc/dist_rakudo.txt | 2 + etc/modules.txt | 69 +++++++++++++++++++++++++ lib/actions/clean.bash | 11 ++++ lib/actions/dist.bash | 48 ++++++++++++++++++ lib/actions/fetch.bash | 98 +++++++++++++++++++++++++++++++++++ lib/actions/install.bash | 127 ++++++++++++++++++++++++++++++++++++++++++++++ lib/actions/test.bash | 6 +++ lib/install-module.raku | 28 ++++++++++ lib/logging.bash | 60 ++++++++++++++++++++++ lib/main.bash | 129 +++++++++++++++++++++++++++++++++++++++++++++++ lib/util.bash | 118 +++++++++++++++++++++++++++++++++++++++++++ 15 files changed, 822 insertions(+) create mode 100644 README.md create mode 100755 bin/rstar create mode 100644 etc/dist_moarvm.txt create mode 100644 etc/dist_nqp.txt create mode 100644 etc/dist_rakudo.txt create mode 100644 etc/modules.txt create mode 100644 lib/actions/clean.bash create mode 100644 lib/actions/dist.bash create mode 100644 lib/actions/fetch.bash create mode 100644 lib/actions/install.bash create mode 100644 lib/actions/test.bash create mode 100644 lib/install-module.raku create mode 100644 lib/logging.bash create mode 100644 lib/main.bash create mode 100644 lib/util.bash diff --git a/README.md b/README.md new file mode 100644 index 0000000..e68318d --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# Rakudo Star + +A user-friendly distribution of the Raku programming language. + +## Quickstart + +After downloading and extracting the tarball (or cloning the git repository), +run `./bin/rstar install`. Follow any on-screen instructions as they appear. +That is all! + +If you happen to find any bugs, please refer to the **Bugs, Feedback and +Patches** section later on in this document to find out how you can get help. + +## Advanced usage + +This section is intended for maintainers of the Rakudo Star distribution. + +### The `rstar` utility + +To help maintainers build the distribution tarball, and end-users to make +effective use of the tarball, a utility has been created, called `rstar`. This +utility depends on the `bash` shell being available. Run it with `-h` to see +what it can do. + +Depending on what action you're trying to run, additional dependencies may be +required. If any of these are missing, `rstar` will throw an error about it. + +#### Exit codes + +- ` 1` - die() was encountered. This is always a bug; +- ` 2` - The program was invoked incorrectly; +- ` 3` - Some required dependencies are missing. + +#### Environment Variables + +The `rstar` utility can be affected by environment variables. These may help +out when debugging issues. + +- `RSTAR_DEBUG` - If set to a non-null value, additional debugging output will + magically appear; +- `RSTAR_MESSY` - If set to a non-null value, the `tmp` directory will not be + cleaned when `rstar` exits. + +### Community Modules + +One of Rakudo Star's main features is in supplying users with a number of +popular community modules. This section details the mechanics of how these are +included. + +*You should always prefer to use a pinned version of a module!* + +#### modules.txt + +This file contains references to all community modules to be bundled with +Rakudo Star. It is a space-separated format. The first column is the name of +the module, the second the protocol to use, with the third column being the +URL to fetch it from. Columns following the third have different meaning +depending on the protocol. + +##### `git` + +The git protocol clones a single ref, with a depth of 1. Which ref is going to +be cloned is specified in the 4th column of its `modules.txt` entry. After +cloning, the `.git` directory is removed. + +##### `http + +The http protocol is the most straightforward, it downloads a tarball +(`.tar.gz`) and unpacks it. If a value is specified in the 4th column of the +entry, this will be used as prefix, and will be stripped away when the +extracted sources are moved into the `dist` directory. + +### Quickstart to Releasing Rakudo Star + +Your first step will be to prepare a new tarball. + + rstar clean # Clean up old sources + $EDITOR etc/dist_moarvm.txt # Update values as necessary + $EDITOR etc/dist_nqp.txt # Update values as necessary + $EDITOR etc/dist_rakudo.txt # Update values as necessary + $EDITOR etc/modules.txt # Update values as necessary + rstar fetch # Download new sources + rstar install # Compile and install Rakudo Star + rstar test # Run tests + rstar dist # Create a new distribution tarball + +Once you have a tarball, you should upload it to be available to others. Common +places include: + +- rakudo.org (ask around in #raku-dev for someone to help you if needed); +- Your personal website. + +Next up, you will have to tell people of the new distribution tarball existing. +There are several places to announce this at. The most "official" one would be +the `perl6-compiler@perl.org` mailing list. The `perl6-users@perl.org` mailing +list is also a good choice, as are public places such as Reddit. + +## Bugs, Feedback and Patches + +Bugs, feedback or patches for this project can be sent to +`p.spek+rakudo-star@tyil.work`. Alternatively, you can reach out to `tyil` on +Freenode, DareNET or Matrix. + +## License + +The software in this repository is distributed under the terms of the Artistic +License 2.0, unless specified otherwise. diff --git a/bin/rstar b/bin/rstar new file mode 100755 index 0000000..1049cd9 --- /dev/null +++ b/bin/rstar @@ -0,0 +1,15 @@ +#!/bin/sh + +export BASEDIR=$(CDPATH="" cd -- "$(dirname -- "$0")/.." && pwd -P) + +main() { + if ! command -v bash > /dev/null + then + printf "You need bash to run rstar\n" >&2 + exit 1 + fi + + exec bash "$BASEDIR/lib/main.bash" "$@" +} + +main "$@" diff --git a/etc/dist_moarvm.txt b/etc/dist_moarvm.txt new file mode 100644 index 0000000..eb62af2 --- /dev/null +++ b/etc/dist_moarvm.txt @@ -0,0 +1,2 @@ +url=https://www.moarvm.org/releases/MoarVM-%s.tar.gz +version=2020.01.1 diff --git a/etc/dist_nqp.txt b/etc/dist_nqp.txt new file mode 100644 index 0000000..daf0962 --- /dev/null +++ b/etc/dist_nqp.txt @@ -0,0 +1,2 @@ +url=https://github.com/perl6/nqp/releases/download/%s/nqp-%s.tar.gz +version=2020.01 diff --git a/etc/dist_rakudo.txt b/etc/dist_rakudo.txt new file mode 100644 index 0000000..2000f5c --- /dev/null +++ b/etc/dist_rakudo.txt @@ -0,0 +1,2 @@ +url=https://github.com/rakudo/rakudo/releases/download/%s/rakudo-%s.tar.gz +version=2020.01 diff --git a/etc/modules.txt b/etc/modules.txt new file mode 100644 index 0000000..4cb5199 --- /dev/null +++ b/etc/modules.txt @@ -0,0 +1,69 @@ +# Zef, the package manager +zef git https://github.com/ugexe/zef.git v0.8.3 + +# Documentation utilities +rakudoc git https://github.com/Raku/doc.git master + +# Config +Hash-Merge git https://github.com/scriptkitties/p6-Hash-Merge.git v1.0.0 +Config git https://github.com/scriptkitties/p6-Config.git v2.0.1 + +# Taken from the previous iteration of Rakudo Star +URI git https://github.com/raku-community-modules/uri.git master +JSON::Fast +JSON::Name +JSON::Unmarshal +JSON::Marshal +JSON::Class +META6 +License::SPDX +Test::META +XML::Writer +SVG +SVG::Plot +Terminal::ANSIColor +OO::Monitors +Test::Mock +Grammar::Profiler::Simple +Grammar::Debugger +MIME::Base64 +JSON::Tiny +OpenSSL +IO::Socket::SSL +LWP::Simple +Digest::MD5 +HTTP::Status +Template::Mustache +PSGI +HTTP::Easy +Template::Mojo +NativeHelpers::Blob +DBIish +Test::When +File::Directory::Tree +Digest +Testo +Temp::Path +Pod::Load +Test::Output +Pod::To::HTML +Pod::To::BigPage +File::Temp +File::Find +Debugger::UI::CommandLine +File::Which +Shell::Command +LibraryMake +IO::String +DateTime::Format +DateTime::Parse +IO::Capture::Simple +Test::Util::ServerPort +Encode +HTTP::UserAgent +JSON::RPC +Getopt::Long +TAP +App::Prove6 +LibraryCheck +Readline diff --git a/lib/actions/clean.bash b/lib/actions/clean.bash new file mode 100644 index 0000000..c00e85d --- /dev/null +++ b/lib/actions/clean.bash @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +action() { + remove "$BASEDIR/tmp" + remove "$BASEDIR/install" +} + +remove() { + info "Removing $1" + rm -fr -- "$1" +} diff --git a/lib/actions/dist.bash b/lib/actions/dist.bash new file mode 100644 index 0000000..3b06b4f --- /dev/null +++ b/lib/actions/dist.bash @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +RSTAR_DEPS_BIN+=( + git + tar +) + +action() { + local version="${1:-$(date +%Y.%m)}" + WORKDIR="$BASEDIR/tmp/rakudo-star-$version" + + info "Creating distribution contents at $WORKDIR" + + cd -- "$BASEDIR" + + # Include files from this project + for file in $(git ls-files) + do + dist_include "/$file" + done + + # Include the sources of all components + for src in dist/src/* + do + dist_include "/$src" + done + + # Add a MANIFEST.txt + cd -- "$WORKDIR" + find . > MANIFEST.txt + + # Tar it all up into a distribution tarball + info "Creating tarball out of $WORKDIR" + + local tarball="$BASEDIR/dist/rakudo-star-$version.tar.gz" + + mkdir -p -- "$(dirname "$tarball")" + cd -- "$BASEDIR/tmp" + + tar czf "$tarball" "rakudo-star-$version" + + info "Distribution tarball available at $tarball" +} + +dist_include() { + mkdir -p -- "$(dirname "${WORKDIR}$1")" + cp -r -- "${BASEDIR}$1" "${WORKDIR}$1" +} diff --git a/lib/actions/fetch.bash b/lib/actions/fetch.bash new file mode 100644 index 0000000..c489608 --- /dev/null +++ b/lib/actions/fetch.bash @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +RSTAR_DEPS_BIN=( + awk + curl + git + tar +) + +action() { + # Ensure the directory to download to exists + mkdir -p "$BASEDIR/dist/src/core" + + # Download all core components + for component in moarvm nqp rakudo + do + download_core "$component" + done + + mkdir -p "$BASEDIR/dist/src/modules" + + # Download all modules available over http + list_modules "http" | while read -r name proto url prefix + do + download_module_http "$name" "$url" "$prefix" + done + + # Download all modules available over git + list_modules "git" | while read -r name proto url ref + do + download_module_git "$name" "$url" "$ref" + done +} + +download_core() { + local version="$(config_etc_kv "dist_$1.txt" "version")" + local source="$(echo "$(config_etc_kv "dist_$1.txt" "url")" | sed "s/%s/$version/g")" + local destination="$BASEDIR/dist/src/core/$1-$version" + + if [[ -d $destination ]] + then + warn "Skipping sources for $1, destination already exists: $destination" + return 0 + fi + + mkdir -p -- "$destination" + + tarball="$(fetch "$source")" + tar xzf "$tarball" -C "$destination" --strip-components=1 && return + + crit "Failed to download $destination" + rm -f -- "$destination" +} + +download_module_git() { + local name=$1 + local url=$2 + local ref=$3 + local destination="$BASEDIR/dist/src/modules/$name" + + if [[ -d "$destination" ]] + then + warn "Skipping sources for $name, destination already exists: $destination" + return 0 + fi + + notice "Cloning $url@$ref to $destination" + git clone -b "$ref" "$url" --depth=1 --single-branch "$destination" \ + > /dev/null 2>&1 + + rm -fr -- "$destination/.git" +} + +download_module_http() { + local name=$1 + local url=$2 + local prefix=$3 + local destination="$BASEDIR/dist/src/modules/$name" + + if [[ -d "$destination" ]] + then + warn "Skipping sources for $name, destination already exists: $destination" + return 0 + fi + + local tarball="$(fetch "$url")" + local extracted="$(tempdir)" + + notice "Extracting $tarball into $extracted" + tar xzf "$tarball" -C "$extracted" + + notice "Moving $extracted/$prefix to $destination" + mv -- "$extracted/$prefix" "$destination" +} + +list_modules() { + awk '/^[^#]/ && $2 == "'"$1"'" { print }' "$BASEDIR/etc/modules.txt" +} diff --git a/lib/actions/install.bash b/lib/actions/install.bash new file mode 100644 index 0000000..30b88dd --- /dev/null +++ b/lib/actions/install.bash @@ -0,0 +1,127 @@ +#!/usr/bin/bash + +RSTAR_DEPS_BIN+=( + awk + gcc + make + perl +) + +RSTAR_DEPS_PERL+=( + ExtUtils::Command + Pod::Usage +) + +action() { + local OPTIND + + while getopts ":b:p:" opt + do + case "$opt" in + b) RSTAR_BACKEND=$OPTARG ;; + p) RSTAR_PREFIX=$OPTARG ;; + esac + done + + shift $(( OPTIND -1 )) + # TODO: Check if binaries are available + + mkdir -p -- "$RSTAR_PREFIX" + local prefix_absolute="$(CDPATH="" cd -- "$RSTAR_PREFIX" && pwd -P)" + + info "Installing Raku in $prefix_absolute" + + # Compile all core components + for component in moarvm nqp rakudo + do + VERSION="$(config_etc_kv "dist_$component.txt" "version")" \ + build_"$component" \ + --prefix="$RSTAR_PREFIX" \ + --relocatable \ + && continue + + die "Build failed!" + done + + # Install community modules + failed_modules=() + + for module in $(awk '/^[^#]/ {print $1}' "$BASEDIR/etc/modules.txt") + do + info "Installing $module" + + install_raku_module "$BASEDIR/dist/src/modules/$module" \ + && continue + + failed_modules+=("$module") + done + + # Show a list of all modules that failed to install + if [[ $failed_modules ]] + then + crit "The following modules failed to install:" + + for module in "${failed_modules[@]}" + do + crit " $module" + done + fi + + # Friendly message + info "Rakudo Star has been installed into $prefix_absolute!" + info "You may need to add the following paths to your \$PATH:" + info " $prefix_absolute/bin" + info " $prefix_absolute/share/perl6/site/bin" + info " $prefix_absolute/share/perl6/vendor/bin" + info " $prefix_absolute/share/perl6/core/bin" +} + +build_moarvm() { + info "Starting build on MoarVM" + + build_prepare "$BASEDIR/dist/src/core/moarvm-$VERSION" || return + perl Configure.pl \ + "$@" \ + && make \ + && make install \ + || return +} + +build_nqp() { + info "Starting build on NQP" + + build_prepare "$BASEDIR/dist/src/core/nqp-$VERSION" || return + perl Configure.pl \ + --backend="$RSTAR_BACKEND" \ + "$@" \ + && make \ + && make install \ + || return +} + +build_rakudo() { + info "Starting build on Rakudo" + + build_prepare "$BASEDIR/dist/src/core/rakudo-$VERSION" || return + perl Configure.pl \ + --backend="$RSTAR_BACKEND" \ + "$@" \ + && make \ + && make install \ + || return +} + +build_prepare() { + local source="$1" + local destination="$(tempdir)" + + notice "Using $destination as working directory" + + cp -R -- "$source/." "$destination" \ + && cd -- "$destination" \ + || return +} + +install_raku_module() { + "$RSTAR_PREFIX/bin/raku" "$BASEDIR/lib/install-module.raku" "$1" +} diff --git a/lib/actions/test.bash b/lib/actions/test.bash new file mode 100644 index 0000000..68c6d60 --- /dev/null +++ b/lib/actions/test.bash @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +action() { + # TODO: Implement test + die "Not yet implemented" +} diff --git a/lib/install-module.raku b/lib/install-module.raku new file mode 100644 index 0000000..8e70ab2 --- /dev/null +++ b/lib/install-module.raku @@ -0,0 +1,28 @@ +#!/usr/bin/env raku + +use v6.d; + +#| Install a Raku module. +sub MAIN ( + #| The path to the Raku module sources. + IO() $path is copy, + + #| The repository to install it in. Options are "site" (ment for + #| user-installed modules), "vendor" (ment for distributions that want + #| to include more modules) and "core" (ment for modules distributed + #| along with Raku itself). + Str:D :$repo = 'vendor', + + #| Force installation of the module. + Bool:D :$force = True, +) { + CATCH { + default { $_.say; exit 1; } + } + + my $repository = CompUnit::RepositoryRegistry.repository-for-name($repo); + my $meta-file = $path.add('META6.json'); + my $dist = Distribution::Path.new($path, :$meta-file); + + $repository.install($dist, :$force); +} diff --git a/lib/logging.bash b/lib/logging.bash new file mode 100644 index 0000000..33ead42 --- /dev/null +++ b/lib/logging.bash @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# TODO: Only use colors on terminals that are known to work well with them. + +# The base function to output logging information. This should *not* be used +# directly, but the helper functions can be used safely. +log() { + local OPTIND + local color + local format="[%s] %s\n" + + while getopts ":c:" opt + do + case "$opt" in + c) color=$OPTARG ;; + esac + done + + shift $(( OPTIND - 1 )) + + printf "$color[%s] %s\e[0m\n" "$(date +%FT%T)" "$*" >&2 +} + +debug() { + [[ -z $RSTAR_DEBUG ]] && return + log -c "\e[1;30m" -- "$*" +} + +info() { + log -- "$*" +} + +notice() { + log -c "\e[0;34m" -- "$*" +} + +warn() { + log -c "\e[0;33m" -- "$*" +} + +crit() { + log -c "\e[0;31m" -- "$*" +} + +alert() { + log -c "\e[1;31m" -- "$*" +} + +emerg() { + log -c "\e[1;4;31m" -- "$*" +} + +export -f log +export -f debug +export -f info +export -f notice +export -f warn +export -f crit +export -f alert +export -f emerg diff --git a/lib/main.bash b/lib/main.bash new file mode 100644 index 0000000..1d57dcc --- /dev/null +++ b/lib/main.bash @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +source "$(dirname "$BASH_SOURCE")/util.bash" +source "$(dirname "$BASH_SOURCE")/logging.bash" + +main() { + [[ -z $1 ]] && usage && exit 2 + + local action="$1" + shift + + debug "Handling action '$action'" + + local action_path="$BASEDIR/lib/actions/$action.bash" + + debug "Checking $action_path" + + if [[ ! -f $action_path ]] + then + debug "No script found to handle action, showing usage" + usage + exit 2 + fi + + # Set some global defaults + RSTAR_TOOLS=() + RSTAR_BACKEND=moar + RSTAR_PREFIX="$BASEDIR" + + # Source the file defining the action. + debug "Sourcing $action_path" + source "$action_path" + + # Ensure all required tools are available + depcheck_bin || exit 3 + depcheck_perl || exit 3 + + # TODO: Figure out which OS/distro we're on, to allow for working + # around edge-cases. Probably expose this info as RSTAR_PLATFORM, in an + # associative array. + + # Maintain our own tempdir + export TMPDIR="$BASEDIR/tmp" + mkdir -p -- "$TMPDIR" + debug "\$TMPDIR set to $TMPDIR" + + # Actually perform the action + debug "Running action" + action "$@" + local action_exit=$? + + # Clean up if necessary + if [[ -z $RSTAR_MESSY ]] + then + debug "Cleaning up tempfiles at $TMPDIR" + rm -rf -- "$TMPDIR" + fi + + # Use the action's exit code + exit $action_exit +} + +usage() { + cat < [options] [arguments] + +rstar is the entry point for all utilities to deal with Rakudo Star. + +Actions: + clean Clean up the repository. + dist Create a distributable tarball of this repository. + fetch Fetch all required sources. + install Install Raku on this system. + test Run tests on Raku and the bundled ecosystem modules. +EOF +} + +# This function checks for the availability of (binary) utilities in the user's +# $PATH environment variable. +depcheck_bin() { + local missing=() + + for tool in "${RSTAR_DEPS_BIN[@]}" + do + command -v "$tool" > /dev/null && continue + + missing+=("$tool") + done + + if [[ $missing ]] + then + alert "Some required tools are missing:" + + for tool in "${missing[@]}" + do + # TODO: Include current distro's package name + # containing the tool + alert " $tool" + done + + return 1 + fi +} + +# This function checks for the availability of all Perl modules required. +depcheck_perl() { + local missing=() + + for module in "${RSTAR_DEPS_PERL[@]}" + do + perl -M"$module" -e 0 2> /dev/null && continue + + missing+=("$tool") + done + + if [[ $missing ]] + then + alert "Some required Perl modules are missing:" + + for modules in "${missing[@]}" + do + alert " $module" + done + + return 1 + fi +} + +main "$@" diff --git a/lib/util.bash b/lib/util.bash new file mode 100644 index 0000000..025aeba --- /dev/null +++ b/lib/util.bash @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# Log a message as error, and exit the program. This is intended for serious +# issues that prevent the script from running correctly. The exit code can be +# specified with -i, or will default to 1. +die() { + local OPTIND + local code + + while getopts ":i:" opt + do + case "$opt" in + i) code=$OPTARG ;; + esac + done + + shift $(( OPTIND -1 )) + + alert "$@" + exit ${code:-1} +} + +# Fetch a file from an URL. Using this function introduces a dependency on curl. +fetch() { + local OPTIND + local buffer + + while getopts ":o:" opt + do + case "$opt" in + o) buffer=$OPTARG ;; + esac + done + + shift $(( OPTIND -1 )) + + [[ -z $buffer ]] && buffer="$(tempfile)" + + notice "Downloading $1 to $buffer" + + # TODO: Switch to the most appropriate downloading tool, depending on + # what is available. + + curl -Ls "$1" > "$buffer" + local exit_code=$? + + printf "%s" "$buffer" + + return $exit_code +} + +# Read a particular value from a key/value configuration file. Using this +# function introduces a dependency on awk. +config_etc_kv() { + local file="$BASEDIR/etc/$1" + shift + + if [[ ! -f $file ]] + then + crit "Tried to read value for $1 from $file, but $file does not exist" + return + fi + + debug "Reading value for $1 from $file" + + awk -F= '$1 == "'"$1"'" { print $NF }' "$file" +} + +# Create a temporary directory. Similar to tempfile, but you'll get a directory +# instead. +tempdir() { + local dir="$(mktemp -d)" + + # Ensure the file was created succesfully + if [[ ! -d "$dir" ]] + then + die "Failed to create a temporary directory at $dir" + fi + + debug "Temporary file created at $dir" + + printf "$dir" +} + +# Create a temporary file. In usage, this is no different from mktemp itself, +# however, it will apply additional checks to ensure everything is going +# correctly, and the files will be cleaned up automatically at the end. +tempfile() { + local OPTIND + local extension="tmp" + + while getopts ":x:" opt + do + case "$opt" in + x) extension=$OPTARG ;; + esac + done + + shift $(( OPTIND -1 )) + + local file="$(mktemp --suffix ".$extension")" + + # Ensure the file was created succesfully + if [[ ! -f "$file" ]] + then + die "Failed to create a temporary file at $file" + fi + + debug "Temporary file created at $file" + + printf "$file" +} + +export -f config_etc_kv +export -f die +export -f fetch +export -f tempdir +export -f tempfile -- cgit v1.1