v0.0.5

Vonng · Aug 19, 2020 · 0fe9e82 · 0fe9e82
1 parent 04053fa
commit 0fe9e82
Show file tree

Hide file tree

Showing 9 changed files with 196 additions and 166 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@ vagrant/.vagrant
 # packages cache
 pkg/
 pkg.tar.gz
+pkg.tgz
 
 # docs are not ready
 doc/

diff --git a/Makefile b/Makefile
@@ -5,15 +5,24 @@ default: start
 # Public objective
 ###############################################################
 # create a new cluster
-new: clean start init
+new: clean start upload init
 
 # write dns record to your own host, sudo required
 dns:
 	if ! grep --quiet "pigsty dns records" /etc/hosts ; then cat files/pigsty_dns >> /etc/hosts; fi
 
-# copy yum packages to your own host, pigsty/pkg
+# cache / upload rpm packages (this is useful for accelerate or perform offline installation)
 cache:
-	rm -rf pkg/* && mkdir -p pkg && scp -r meta:/www/pigsty/* pkg/
+	rm -rf pkg/* && mkdir -p pkg;
+	ssh -t meta "sudo tar -zcf /tmp/pkg.tgz -C /www pigsty; sudo chmod a+r /tmp/pkg.tgz"
+	scp -r meta:/tmp/pkg.tgz pkg.tgz
+	ssh -t meta "sudo rm -rf /tmp/pkg.tgz"
+
+# upload rpm cache to meta node
+upload:
+	ssh -t meta "sudo rm -rf /tmp/pkg.tgz"
+	scp -r pkg.tgz meta:/tmp/pkg.tgz
+	ssh -t meta "sudo mkdir -p /www/pigsty/; sudo rm -rf /www/pigsty/*; sudo tar -xf /tmp/pkg.tgz --strip-component=1 -C /www/pigsty/"
 
 # init will pull up entire cluster
 init:
@@ -34,6 +43,8 @@ clean:
 	cd vagrant && vagrant destroy -f --parallel; exit 0
 up:
 	cd vagrant && vagrant up
+mt:
+	cd vagrant && vagrant up meta
 halt:
 	cd vagrant && vagrant halt
 status:
@@ -72,13 +83,20 @@ rw2:
 	while true; do pgbench -nv -P1 -c2 -T10 postgres://test:test@pg-test:5433/test; done
 ro2:
 	while true; do pgbench -nv -P1 -c8 -T10 --select-only postgres://test:test@pg-test:5434/test; done
+rl:
+	ssh -t node-1 "sudo -iu postgres patronictl -c /pg/bin/patroni.yml list -W"
+r1:
+	ssh -t node-1 "sudo reboot"
+r2:
+	ssh -t node-2 "sudo reboot"
+r3:
+	ssh -t node-3 "sudo reboot"
+ckpt:
+	ansible all -b --become-user=postgres -a "psql -c 'CHECKPOINT;'"
 gis:
 	# psql postgres://test:test@pg-test:5433/test -c 'CREATE EXTENSION postgis;'
 	psql postgres://test:test@pg-test:5433/test -f files/adcode.sql
 
-ckpt:
-	ansible all -b --become-user=postgres -a "psql -c 'CHECKPOINT;'"
-
 ###############################################################
 # grafna management
 ###############################################################

diff --git a/local.repo b/local.repo
diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml
@@ -2,30 +2,33 @@
 #------------------------------------------------------------------------------
 # Grafana
 #------------------------------------------------------------------------------
-grafana_force_download_plugins: false         # force redownload grafana plugins
-grafana_time_interval: 2s                     # force redownload grafana plugins
-grafana_provision_via_db: true                # provision via copy sqlite db by default
-grafana_dashboards: []                        # default dashboards
-grafana_plugins_force_download: false         # force re-download grafana plugins ?
-grafana_plugins_cache: /tmp/plugins.tar.gz    # path to grafana plugins tarball
-grafana_plugins:                              # default grafana plugins list
+grafana_url: http://admin:admin@localhost:3000 # grafana url
+grafana_plugin: install                        # none|install|reinstall
+grafana_cache: /tmp/plugins.tar.gz             # path to grafana plugins tarball
+grafana_provision_mode: db                     # none|db|api
+
+grafana_plugins:                               # default grafana plugins list
   - redis-datasource
   - simpod-json-datasource
+  - fifemon-graphql-datasource
   - sbueringer-consul-datasource
   - camptocamp-prometheus-alertmanager-datasource
+  - ryantxu-ajax-panel
   - marcusolsson-hourly-heatmap-panel
   - michaeldmoore-multistat-panel
   - marcusolsson-treemap-panel
+  - pr0ps-trackmap-panel
   - dalvany-image-panel
-  - ryantxu-ajax-panel
+  - magnesium-wordcloud-panel
   - cloudspout-button-panel
+  - speakyourcode-button-panel
   - jdbranham-diagram-panel
   - grafana-piechart-panel
   - snuids-radar-panel
   - digrich-bubblechart-panel
 
-# - reference : dcs metadata - #
-dcs_type: consul                  # default dcs server type: consul
-consul_check_interval: 15s        # default service check interval
-consul_check_timeout:  1s         # default service check timeout
+grafana_git_plugins:
+  - https://github.com/Vonng/grafana-echarts
+
+grafana_dashboards: []                        # default dashboards
 ...
diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml
@@ -1,108 +1,114 @@
 ---
-#==============================================================#
-# Setup grafana
-#==============================================================#
-- name: Setup Grafana
+
+#--------------------------------------------------------------#
+# 1. Install Grafana RPM
+#--------------------------------------------------------------#
+- name: Make sure grafana is installed
   tags: grafana_install
+  package: name=grafana state=present
+
+#--------------------------------------------------------------#
+# 2. Install Plugins
+#--------------------------------------------------------------#
+- name: Install grafana plugins
+  tags: grafana_plugin
+  when: grafana_plugin != 'none'
   block:
-    #==============================================================#
-    # Install grafana
-    #==============================================================#
-    - name: Make sure grafana is installed
-      package: name=grafana state=present
 
-    #==============================================================#
-    # Install grafana plugins: check cache exists
-    #==============================================================#
+    #--------------------------------------------------------------#
+    # Check Cache Exists
+    #--------------------------------------------------------------#
     - name: Check grafana plugin cache exists
-      stat: path=/www/pigsty/grafana/plugins.tar.gz
+      stat: path={{ grafana_cache }}
       register: grafana_plugins_cache
 
-    #==============================================================#
-    # Install grafana plugins from cache when applicable
-    #==============================================================#
-    # extract cache pacakge and install
-    - name: Provision grafana plugin via cache
+
+    #--------------------------------------------------------------#
+    # Extract plugins cache if applicable
+    #--------------------------------------------------------------#
+    - name: Provision grafana plugins via cache
       when: grafana_plugins_cache.stat.exists
       shell: |
         rm -rf /var/lib/grafana/plugins
-        tar -xf {{ grafana_plugins_cache }} -C /var/lib/grafana
+        tar -xf {{ grafana_cache }} -C /var/lib/grafana
         chown -R grafana /var/lib/grafana
 
-    #==============================================================#
-    # Install grafana plugins and make cache
-    #==============================================================#
+    #--------------------------------------------------------------#
+    # download plugins from web if 1. cache not exists, 2. reinstall mode
+    #--------------------------------------------------------------#
     - name: Download grafana plugins from web
-      when: not grafana_plugins_cache.stat.exists or grafana_plugins_force_download
-      ignore_errors: true   # don't let these little shit ruin the entire procedure
+      when: not grafana_plugins_cache.stat.exists or grafana_plugin == 'reinstall'
+      ignore_errors: true
       environment: "{{ proxy_env | default({}) }}"
       grafana_plugin: name={{ item }}
       with_items: "{{ grafana_plugins  }}"
 
-    - name: Create grafana plugins cache
+    - name: Download grafana plugins from web
+      when: not grafana_plugins_cache.stat.exists or grafana_plugin == 'reinstall'
       ignore_errors: true
-      when: not grafana_plugins_cache.stat.exists or grafana_plugins_force_download
+      environment: "{{ proxy_env | default({}) }}"
+      git:
+        repo: '{{ item }}'
+        dest: /var/lib/grafana/plugins/{{ item | basename }}
+        update: yes
+      with_items: "{{ grafana_git_plugins }}"
+
+    #--------------------------------------------------------------#
+    # Make new cache file after download
+    #--------------------------------------------------------------#
+    - name: Create grafana plugins cache
+      when: not grafana_plugins_cache.stat.exists or grafana_plugin == 'reinstall'
       shell: |
-        mkdir -p /www/pigsty/grafana; tar -zcf {{ grafana_plugins_cache }} -C /var/lib/grafana/ plugins
+        mkdir -p {{ grafana_cache | dirname }};
+        tar -zcf {{ grafana_cache }} -C /var/lib/grafana/ plugins
 
 
-    #==============================================================#
-    # Config grafana
-    #==============================================================#
+#--------------------------------------------------------------#
+# Config grafana
+#--------------------------------------------------------------#
+- name: Config Grafana
+  tags: grafana_config
+  block:
+    # main config
     - name: Copy /etc/grafana/grafana.ini
       copy: src=grafana/grafana.ini dest=/etc/grafana/grafana.ini owner=grafana group=grafana
 
-    #==============================================================#
-    # Launch grafana
-    #==============================================================#
+    # sqlite db provision
+    - name: Copy grafana.db to data dir
+      copy: src=grafana/grafana.db dest=/var/lib/grafana/grafana.db owner=grafana mode=0600
+
+
+#--------------------------------------------------------------#
+# Launch grafana
+#--------------------------------------------------------------#
+- name: Launch Grafana
+  tags: grafana_launch
+  block:
     - name: Launch grafana service
       systemd: name=grafana-server state=restarted enabled=yes daemon_reload=yes
 
     - name: Wait for grafana online
       wait_for: host=localhost port=3000 state=started
 
-    #==============================================================#
-    # Register grafana consul service
-    #==============================================================#
+
+- name: Register Grafana Service
+  tags: grafana_register
+  block:
+    #--------------------------------------------------------------#
+    # Register grafana
+    #--------------------------------------------------------------#
     - name: Register consul grafana service
       template: src=svc-grafana.json.j2 dest=/etc/consul.d/svc-grafana.json owner=consul group=admin mode=0660
 
     - name: Reload consul
       systemd: name=consul state=reloaded enabled=yes daemon_reload=yes
 
 
-
-
-
-#==============================================================#
-# 1. Provision grafana by copy sqlite db
-#==============================================================#
-- name: Provision Grafana via db
-  when: grafana_provision_via_db
-  tags: grafana_provision
-  block:
-    #==============================================================#
-    # Copy grafana.db
-    #==============================================================#
-    - name: Launch meta grafana service
-      systemd: name=grafana-server state=stopped enabled=yes daemon_reload=yes
-
-    - name: Copy grafana.db to data dir
-      copy: src=grafana/grafana.db dest=/var/lib/grafana/grafana.db owner=grafana mode=0600
-
-    - name: Restart meta grafana service
-      systemd: name=grafana-server state=restarted enabled=yes daemon_reload=yes
-
-    - name: Wait for meta grafana online
-      wait_for: host=localhost port=3000 state=started
-
-
-
-#==============================================================#
-# 2. Provision grafana by calling API
-#==============================================================#
+#--------------------------------------------------------------#
+# Provision Grafana
+#--------------------------------------------------------------#
 - name: Provision Grafana via API
-  when: not grafana_provision_via_db
+  when: grafana_provision_mode == 'api'
   tags: grafana_provision
   block:
     #==============================================================#
@@ -121,7 +127,7 @@
     - name: Provision prometheus datasource
       grafana_datasource:
         name: "Prometheus"
-        grafana_url: "http://{{ grafana_domain_name }}"
+        grafana_url: "{{ grafana_url }}"
         grafana_user: "admin"
         grafana_password: "admin"
         ds_type: "prometheus"
@@ -134,7 +140,7 @@
     #==============================================================#
     - name: Provision grafana dashboards
       grafana_dashboard:
-        grafana_url: "http://{{ grafana_domain_name }}"
+        grafana_url: "{{ grafana_url }}"
         grafana_user: "admin"
         grafana_password: "admin"
         state: present

diff --git a/roles/node/defaults/main.yml b/roles/node/defaults/main.yml
@@ -21,20 +21,20 @@ node_repo_url: []                             # additional repo to be installed
 node_local_repo: pigsty                       # if provide, all default repo will be disabled except this
 node_repo_config: []                          # commands to config yum repos
 
-# - node packages - #
+# - node pkgs - #
 node_packages:                                # common packages for all nodes
-  - wget yum-utils ntp uuid lz4 nc pv jq vim make patch bash lsof wget unzip git readline zlib openssl
-  - numactl grubby sysstat dstat iotop bind-utils net-tools tcpdump socat ipvsadm telnet ca-certificates
-  - python-pip python-ipython python-psycopg2 python3-psycopg2
-
-node_extra_packages:                          # extra packages for nodes
-  - node_exporter pg_exporter postgres_exporter blackbox_exporter     # exporter
-  - consul consul-template etcd haproxy keepalived                    # dcs & proxy
-  # - docker-ce docker-ce-cli rkt                                     # container  (optional)
-  # - kubelet kubectl kubeadm kubernetes-cni helm                     # kubernetes (optional)
-
+  - wget,yum-utils,ntp,uuid,lz4,vim-minimal,make,patch,bash,lsof,wget,unzip,git,readline,zlib,openssl
+  - numactl,grubby,sysstat,dstat,iotop,bind-utils,net-tools,tcpdump,socat,ipvsadm,telnet,tuned,pv,jq
+  - python-pip,python-psycopg2
+  - node_exporter,pg_exporter
+  - consul,consul-template,etcd,haproxy,keepalived
+
+  # - postgresql12*
+node_extra_packages: []                       # extra packages for nodes
 node_meta_packages:                           # packages for meta nodes only
-  - nginx dnsmasq ansible grafana prometheus2 pushgateway alertmanager nginx_exporter blackbox_exporter pgbadger
+  - grafana,prometheus2,alertmanager,nginx_exporter,blackbox_exporter,pushgateway
+  - dnsmasq,nginx,ansible,pgbadger
+
 
 # - node features - #
 node_disable_numa: false                      # disable numa, important for production database, reboot required

diff --git a/roles/repo/defaults/main.yml b/roles/repo/defaults/main.yml
@@ -12,7 +12,7 @@ repo_address: yum.pigsty                      # local repo host (ip or hostname,
 repo_port: 80                                 # repo server listen address, must same as repo_address!
 repo_home: /www                               # default repo dir location
 repo_rebuild: false                           # force re-download packages
-
+repo_exist: false
 
 # - upstream repo - #
 repo_remove: true                             # remove existing repos
@@ -117,18 +117,18 @@ repo_upstreams:                               # additional repos to be installed
 
 # - repo packages- #
 repo_packages:
-  - epel-release nginx wget yum-utils                                                     # bootstrap packages
-  - ntp uuid lz4 nc pv jq vim make patch bash lsof wget unzip git                         # basic system util
+  - epel-release nginx wget yum-utils yum createrepo                                      # bootstrap packages
+  - ntp uuid lz4 nc pv jq vim-enhanced make patch bash lsof wget unzip git tuned          # basic system util
   - readline zlib openssl libyaml libxml2 libxslt perl-ExtUtils-Embed ca-certificates     # basic pg dependency
   - numactl grubby sysstat dstat iotop bind-utils net-tools tcpdump socat ipvsadm telnet  # system utils
   - grafana prometheus2 pushgateway alertmanager                                          # monitor and ui
   - node_exporter postgres_exporter nginx_exporter blackbox_exporter                      # exporter
   - consul consul_exporter consul-template etcd                                           # dcs
-  - ansible python python-pip python-ipython python-psycopg2                              # ansible & python
+  - ansible python python-pip python-psycopg2                                             # ansible & python
   - python3 python3-psycopg2                                                              # python3
   - haproxy keepalived dnsmasq                                                            # proxy and dns
-  - docker-ce docker-ce-cli rkt                                                           # container
-  - kubelet kubectl kubeadm kubernetes-cni helm                                           # kubernetes
+  # - docker-ce docker-ce-cli rkt                                                         # container
+  # - kubelet kubectl kubeadm kubernetes-cni helm                                         # kubernetes
   - postgresql12* postgis30_12* timescaledb_12 citus_12 pglogical_12                      # postgres 12 basic
   - pg_qualstats12 pg_cron_12 pg_top12 pg_repack12 pg_squeeze12 pg_stat_kcache12 wal2json12 pgpool-II-12 pgpool-II-12-extensions python3-psycopg2 python2-psycopg2
   - ddlx_12 bgw_replstatus12 count_distinct12 extra_window_functions_12 geoip12 hll_12 hypopg_12 ip4r12 jsquery_12 multicorn12 osm_fdw12 mysql_fdw_12 ogr_fdw12 mongo_fdw12 hdfs_fdw_12 cstore_fdw_12 wal2mongo12 orafce12 pagila12 pam-pgsql12 passwordcheck_cracklib12 periods_12 pg_auto_failover_12 pg_bulkload12 pg_catcheck12 pg_comparator12 pg_filedump12 pg_fkpart12 pg_jobmon12 pg_partman12 pg_pathman12 pg_track_settings12 pg_wait_sampling_12 pgagent_12 pgaudit14_12 pgauditlogtofile-12 pgbconsole12 pgcryptokey12 pgexportdoc12 pgfincore12 pgimportdoc12 pgmemcache-12 pgmp12 pgq-12 pgrouting_12 pgtap12 plpgsql_check_12 plr12 plsh12 postgresql_anonymizer12 postgresql-unit12 powa_12 prefix12 repmgr12 safeupdate_12 semver12 slony1-12 sqlite_fdw12 sslutils_12 system_stats_12 table_version12 topn_12
@@ -140,6 +140,7 @@ repo_packages:
 repo_url_packages:
   - https://github.com/Vonng/pg_exporter/releases/download/v0.2.0/pg_exporter-0.2.0-1.el7.x86_64.rpm
   - https://github.com/cybertec-postgresql/patroni-packaging/releases/download/1.6.5-1/patroni-1.6.5-1.rhel7.x86_64.rpm
+  - http://guichaz.free.fr/polysh/files/polysh-0.4-1.noarch.rpm
   # - https://github.com/cybertec-postgresql/vip-manager/releases/download/v0.6/vip-manager_0.6-1_amd64.rpm
 
 ...