refactor complete

Makefile

@@ -17,8 +17,8 @@ cache:
 
 # init will pull up entire cluster
 init:
-	./init-meta.yml
-	./init-test.yml
+	./init-meta.yml -i meta
+	./init-cluster.yml -i pg-test
 
 # down will halt all vm (not destroy)
 down: halt
@@ -61,11 +61,11 @@ stop: halt
 # pgbench (init/read-write/read-only)
 ###############################################################
 ri:
-	pgbench -is10 postgres://test:test@pg-test:5555/test
+	pgbench -is10 postgres://test:test@pg-test:5433/test
 rw:
-	while true; do pgbench -nv -P1 -c2 -T10 postgres://test:test@pg-test:5555/test; done
+	while true; do pgbench -nv -P1 -c2 -T10 postgres://test:test@pg-test:5433/test; done
 ro:
-	while true; do pgbench -nv -P1 -c8 -T10 --select-only postgres://test:test@pg-test:5556/test; done
+	while true; do pgbench -nv -P1 -c8 -T10 --select-only postgres://test:test@pg-test:5434/test; done
 ckpt:
 	ansible all -b --become-user=postgres -a "psql -c 'CHECKPOINT;'"
 

bin/clean-grafana-dashboards.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+HOST="http://admin:[email protected]:3000"
+
+[ ! -d dashboards ] && mkdir -p dashboards
+
+for dash in $(curl -sSL -k $HOST/api/search\?query\=\& | jq '.' | grep -i uid | grep -v "folderUid" | awk -F '"uid": "' '{print $2}' | awk -F '"' '{print $1}'); do
+	curl -sSL -X DELETE -k "${HOST}/api/dashboards/uid/${dash}"
+done

bin/dump-grafana-dashboards.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+HOST="http://admin:[email protected]:3000"
+
+[ ! -d dashboards ] && mkdir -p dashboards
+
+for dash in $(curl -sSL -k $HOST/api/search\?query\=\& | jq '.' | grep -i uri | awk -F '"uri": "' '{ print $2 }' | awk -F '"' '{print $1 }'); do
+	curl -sSL -k "${HOST}/api/dashboards/${dash}" >dashboards/$(echo ${dash} | sed 's,db/,,g').json
+done

bin/k8s-token.sh

@@ -0,0 +1 @@
+#!

bin/load-grafana-dashboards.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+HOST="http://admin:[email protected]:3000"
+
+function load_dashboard_dir() {
+	local json_dir=${1-'/tmp/dashboards'}
+	for dash in $(ls ${json_dir}/ | grep '.json$'); do
+		local src=${json_dir}/${dash}
+		local dest=${src}.payload
+		cat >${dest} <<-EOF
+			{"overwrite": false, "dashboard": $(cat $src)}
+		EOF
+		echo curl -sSL -k -X POST "${HOST}/api/dashboards/db" --header '"Content-Type: application/json"' -d @${dest}
+		curl -sSL -k -X POST "${HOST}/api/dashboards/db" --header "Content-Type: application/json" -d @${dest}
+	done
+}
+load_dashboard_dir

dns-dump.yml

@@ -0,0 +1,28 @@
+#!/usr/bin/env ansible-playbook
+---
+#==============================================================#
+# File      :   dashboard-load.yml
+# Mtime     :   2020-06-19
+# Desc      :   load dashboards to meta grafana
+# Path      :   dashboard-load.yml
+# Author    :   Vonng([email protected])
+#==============================================================#
+- name: Load grafana dashboard from meta node
+  connection: local
+  become: no
+  hosts: meta
+  tags: dashboard_load
+  tasks:
+    - name: Pre process dashboards
+      shell: files/grafana/dashboards/reform.py
+
+    - name: Import Grafana dashboards from json
+      grafana_dashboard:
+        grafana_url: "{{ meta_grafana_server }}"
+        grafana_user: "admin"
+        grafana_password: "admin"
+        state: present
+        overwrite: yes
+        path: files/grafana/dashboards/{{ item }}.json
+      with_items: "{{ meta_grafana_dashboards }}"
+...

hosts

@@ -0,0 +1,2 @@
+[meta]
+10.10.10.10 ansible_host=meta seq=1 role=primary lb_role=master

init-cluster.yml

@@ -0,0 +1,64 @@
+#!/usr/bin/env ansible-playbook
+---
+#==============================================================#
+# File      :   init-cluster.yml
+# Mtime     :   2020-06-21
+# Desc      :   init database cluster
+# Path      :   init-cluster.yml
+# Author    :   Vonng([email protected])
+#==============================================================#
+
+
+#==============================================================#
+# This play will init a new database cluster according to inventory
+#==============================================================#
+- name: Init new database cluster
+  become: yes
+  hosts: all
+  gather_facts: true
+  roles:
+    - role: node                            # node provision setup
+      tags: node
+
+    - role: infra                           # repo, pkg, dns, ntp, etc.
+      tags: infra
+
+    - role: dcs                             # dcs: consul/etcd
+      tags: dcs
+
+    # optional kubernetes support
+    - role: cloud                           # docker & kubernetes (optinal)
+      tags: cloud
+      when: infra_cloud_native_support
+
+    # meta db setup (skip replica & haproxy)
+    - role: pg_preflight                    # pre-flight & install
+      tags: preflight
+
+    - role: pg_install                      # install postgres packages
+      tgas: install
+
+    - role: pg_primary                      # init primary instance
+      tags: primary
+      when: role == 'primary'
+
+    # init replica
+    - role: pg_replica                      # init replica instance
+      when: role != 'primary'
+
+    - role: pg_pgbouncer                    # init connection pooler
+      tags: pgbouncer
+
+    - role: pg_patroni                      # init connection pooler
+      tags: patroni
+
+    - role: pg_monitor                      # init monitor system
+      tags: monitor
+
+    - role: haproxy                         # haproxy load balancer
+      tags: haproxy
+
+    - role: keepalived                      # keepalived
+      tags: vip
+
+...

init-meta.yml

@@ -0,0 +1,66 @@
+#!/usr/bin/env ansible-playbook
+---
+#==============================================================#
+# File      :   init-meta.yml
+# Mtime     :   2020-06-20
+# Desc      :   init meta node
+# Path      :   init-meta.yml
+# Author    :   Vonng([email protected])
+# Note:
+#
+# Init a local yum repo powered by a local nginx web server
+# all necessary rpm packages are downloaded to accelerate
+# cluster initialization, or in case that target machine does
+# not have internet access.
+#
+# RPM packages are downloaded to /www/pigsty. And it will skip
+# download if all packages are already downloaded in there.
+#==============================================================#
+- name: Init meta node
+  become: yes
+  hosts: meta
+  gather_facts: true
+  tags: meta
+  roles:
+    # meta node infra provision
+    - role: repo                            # init local yum repo
+      tags: repo
+
+    - role: node                            # node provision setup
+      tags: node
+
+    - role: infra                           # repo, pkg, dns, ntp, etc.
+      tags: infra
+
+    - role: dcs                             # dcs: consul/etcd
+      tags: dcs
+
+    - role: meta                            # meta provision setup
+      tags: ctrl
+
+    # optional kubernetes support
+    - role: cloud                           # docker & kubernetes (optinal)
+      tags: cloud
+      when: infra_cloud_native_support
+
+    # meta db setup (skip replica & haproxy)
+    - role: pg_preflight                    # pre-flight & install
+      tags: preflight
+
+    - role: pg_install                      # install postgres packages
+      tgas: install
+
+    - role: pg_primary                      # init primary instance
+      tags: primary
+      when: role == 'primary'
+
+    - role: pg_pgbouncer                    # init connection pooler
+      tags: pgbouncer
+
+    - role: pg_monitor                      # init monitor system
+      tags: monitor
+
+    - role: keepalived
+      tags: vip
+
+...

pg-test

@@ -0,0 +1,15 @@
+[cluster]
+10.10.10.11 ansible_host=node-1 seq=1 role=primary lb_role=master
+10.10.10.12 ansible_host=node-2 seq=2 role=replica
+10.10.10.13 ansible_host=node-3 seq=3 role=replica
+
+[cluster:vars]
+cluster=pg-test
+default_creation = True
+default_username = test
+default_password = test
+default_database = test
+ha_enabled = true
+lb_enabled = true
+keepalived_vip = 10.10.10.3
+keepalived_nic = eth1

roles/README.md

@@ -0,0 +1,19 @@
+# Ansible roles for pigsty
+
+Ansible roles:
+
+* [repo](repo/)
+* [node](node/)
+* [infra](infra/)
+* [dcs](dcs/)
+* [cloud](cloud/)
+* [meta](meta/)
+* [pg_preflight](pg_preflight/)
+* [pg_install](pg_install/)
+* [pg_primary](pg_primary/)
+* [pg_replica](pg_replica/)
+* [pg_patroni](pg_patroni/)
+* [pg_pgbouncer](pg_pgbouncer/)
+* [haproxy](haproxy/)
+* [keepalived](keepalived/)
+* [pg_monitor](pg_monitor/)