ci: load sample data (#3828)

ietf-tools · Aug 20, 2024 · 82418c8 · 82418c8
1 parent a62106b
commit 82418c8
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 2 deletions.
diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js
@@ -71,6 +71,7 @@ async function main () {
   )
   await fs.copy(path.join(basePath, 'docker/scripts/app-create-dirs.sh'), path.join(releasePath, 'app-create-dirs.sh'))
   await fs.copy(path.join(basePath, 'dev/deploy-to-container/start.sh'), path.join(releasePath, 'start.sh'))
+  await fs.copy(path.join(basePath, 'dev/deploy-to-container/load_messages.sh'), path.join(releasePath, 'load_messages.sh'))
   // await fs.copy(path.join(basePath, 'test/data'), path.join(releasePath, 'test/data'))
   console.info('Updated configuration files.')
 

diff --git a/dev/deploy-to-container/load_messages.sh b/dev/deploy-to-container/load_messages.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# script to load some small lists into the archive
+
+INCLUDE_FILE="rsync_include.txt"
+
+cd /tmp
+
+# create rsync include file
+cat > "$INCLUDE_FILE" <<EOL
+yang-doctors/***
+mtgvenue/***
+curdle/***
+EOL
+
+# sync
+rsync -av --include-from="$INCLUDE_FILE" --exclude='*' rsync.ietf.org::mailman-archive/ ./
+
+# load data
+cd /workspace/backend
+./manage.py load /tmp/yang-doctors/2018-10.mail
+./manage.py load /tmp/curdle/2017-05.mail
+./manage.py load /tmp/mtgvenue/2016-07.mail
diff --git a/dev/deploy-to-container/start.sh b/dev/deploy-to-container/start.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "Creating /test directories..."
+echo "Creating /data directories..."
 for sub in \
     /data/archive \
     /data/incoming \
@@ -41,8 +41,11 @@ until nc -z -w 5 "$ELASTICSEARCH_HOST" 9200; do
   sleep 5
 done
 
-echo "Running Initializing index..."
+echo "Initializing index..."
 ./backend/manage.py init_index
 
+echo "Loading sample data..."
+./load_messages.sh
+
 echo "Starting Mail Archive..."
 ./backend/manage.py runserver 0.0.0.0:8000 --settings=mlarchive.settings.settings_sandbox
diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
@@ -25,13 +25,17 @@ RUN apt-get update --fix-missing && apt-get install -qy \
     memcached \
     netcat \
     nodejs \
+    rsync \
     rsyslog \
     sudo \
     telnet \
     unzip \
     wget \
     zsh
 
+# netcat: for deploy-to-container wait for elasticsearch
+# rsync: for deploy-to-container load_messages
+
 # purge because of vulnerability (see https://www.cvedetails.com/)
 RUN apt-get purge -y imagemagick imagemagick-6-common