Skip to content

Commit e89a39a

Browse files
wileyjwileyj
authored
Merge pull request #5 from moodmosaic/chore/snapshot
merge broad snapshot work into master branch
2 parents 705b703 + 7c89af4 commit e89a39a

18 files changed

Lines changed: 1217 additions & 413 deletions

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1-
logs/*.log
21
docker/chainstate
2+
.current-chainstate-dir
3+

‎Makefile‎

Lines changed: 194 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,214 @@
1-
ifeq (log,$(firstword $(MAKECMDGOALS)))
2-
Arguments := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
1+
COMMANDS := sudo tar zstd getent stress
2+
$(foreach bin,$(COMMANDS),\
3+
$(if $(shell command -v $(bin) 2> /dev/null),$(info),$(error Missing required dependency: `$(bin)`)))
4+
5+
TARGET := $(firstword $(MAKECMDGOALS))
6+
PARAMS := $(filter-out $(TARGET),$(MAKECMDGOALS))
7+
ifeq ($(TARGET),up-genesis)
8+
export CHAINSTATE_DIR := $(PWD)/docker/chainstate/genesis
9+
endif
10+
ifeq ($(TARGET),genesis)
11+
export CHAINSTATE_DIR := $(PWD)/docker/chainstate/genesis
312
endif
13+
14+
15+
## UID and GID are not currently used, but will be in the near future
416
export UID := $(shell getent passwd $$(whoami) | cut -d":" -f 3)
517
export GID := $(shell getent passwd $$(whoami) | cut -d":" -f 4)
618
EPOCH := $(shell date +%s)
719
PWD = $(shell pwd)
20+
# Set a unique project name (used for checking if the network is running)
21+
PROJECT ?= devnet
822
CHAINSTATE_ARCHIVE ?= $(PWD)/docker/chainstate.tar.zstd
923
export CHAINSTATE_DIR ?= $(PWD)/docker/chainstate/$(EPOCH)
1024
export DOCKER_NETWORK ?= stacks
25+
SERVICES := $(shell CHAINSTATE_DIR="" docker compose -f docker/docker-compose.yml --profile=default config --services)
26+
PAUSE_HEIGHT ?= 999999999999
27+
# Used for the stress testing target. modifies how much cpu to consume for how long
28+
STRESS_CORES ?= $(shell cat /proc/cpuinfo | grep processor | wc -l)
29+
STRESS_TIMEOUT ?= 120
30+
31+
# Create the chainstate dir and extract an archive to it when the "up" target is used
32+
$(CHAINSTATE_DIR): /usr/bin/tar /usr/bin/zstd
33+
@mkdir -p $(CHAINSTATE_DIR)
34+
@if [ "$(TARGET)" = "up" ]; then
35+
if [ -f "$(CHAINSTATE_ARCHIVE)" ]; then
36+
sudo tar --same-owner -xf $(CHAINSTATE_ARCHIVE) -C $(CHAINSTATE_DIR) || exit 1
37+
else
38+
@echo "Chainstate archive ($(CHAINSTATE_ARCHIVE)) not found. Exiting"
39+
rm -rf $(CHAINSTATE_DIR)
40+
exit 1
41+
fi
42+
fi
43+
44+
# Boot the network from the local chainstate archive
45+
up: check-not-running | build $(CHAINSTATE_DIR)
46+
@echo "Starting $(PROJECT) network from chainstate archive"
47+
@echo " Chainstate Dir: $(CHAINSTATE_DIR)"
48+
@echo " Chainstate Archive: $(CHAINSTATE_ARCHIVE)"
49+
echo "$(CHAINSTATE_DIR)" > .current-chainstate-dir
50+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) up -d
51+
52+
# Run the network from genesis
53+
genesis: check-not-running | build $(CHAINSTATE_DIR) /usr/bin/sudo
54+
@echo "Starting $(PROJECT) network from genesis"
55+
@if [ -d "$(CHAINSTATE_DIR)/logs" ]; then \
56+
echo " Removing existing genesis chainstate dir: $(CHAINSTATE_DIR)"; \
57+
rm -rf $(CHAINSTATE_DIR)/logs; \
58+
fi
59+
@echo " Chainstate Dir: $(CHAINSTATE_DIR)"
60+
mkdir -p "$(CHAINSTATE_DIR)"
61+
echo "$(CHAINSTATE_DIR)" > .current-chainstate-dir
62+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) up -d
1163

64+
# secondary name to run genesis network
65+
up-genesis: genesis
66+
# secondary name to bring down genesis network
67+
down-genesis: down
1268

13-
$(CHAINSTATE_DIR):
14-
@echo "Creating Chainstate Dir ($(CHAINSTATE_DIR))"
15-
mkdir -p $@
16-
if [ -f "$(CHAINSTATE_ARCHIVE)" -a "$(MAKECMDGOALS)" = "up" ]; then
17-
sudo tar --same-owner -xf $(CHAINSTATE_ARCHIVE) -C $(CHAINSTATE_DIR) || false
69+
# Shut down the network (chainstate and logs will be preserved, but not logs)
70+
down: backup-logs current-chainstate-dir
71+
@echo "Shutting down $(PROJECT) network"
72+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) down
73+
@if [ -f .current-chainstate-dir ]; then \
74+
rm -f .current-chainstate-dir
1875
fi
1976

20-
up: down build | $(CHAINSTATE_DIR)
21-
@echo "Starting stacks from archive at Epoch 3.2"
22-
@echo " CHAINSTATE_DIR: $(CHAINSTATE_DIR)"
23-
@echo " CHAINSTATE_ARCHIVE: $(CHAINSTATE_ARCHIVE)"
24-
@echo " DOCKER_NETWORK: $(DOCKER_NETWORK)"
25-
docker compose -f docker/docker-compose.yml --profile default up -d
77+
# if the network is in a weird state - this target will force kill (bypassing error checks)
78+
down-force:
79+
@echo "Force Shutting down $(PROJECT) network"
80+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) down
81+
@if [ -f .current-chainstate-dir ]; then \
82+
rm -f .current-chainstate-dir
83+
fi
2684

27-
down:
28-
@echo "Shutting down network"
29-
docker compose -f docker/docker-compose.yml --profile default down -v
85+
# Build the images with a cache if present
86+
build: check-not-running
87+
COMPOSE_BAKE=true PWD=$(PWD) docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) build
3088

31-
up-genesis: down build
32-
@echo "Starting stacks from genesis block"
33-
@echo " CHAINSTATE_DIR: $(PWD)/docker/chainstate/genesis"
34-
CHAINSTATE_DIR=$(PWD)/docker/chainstate/genesis docker compose -f docker/docker-compose.yml --profile default up -d
89+
# Build the images without a cache (default uses cache)
90+
build-no-cache: check-not-running
91+
COMPOSE_BAKE=true PWD=$(PWD) docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) build --no-cache
3592

36-
down-genesis: down
93+
# Stream specified service logs to STDOUT. does not validate if PARAMS is supplied
94+
log: current-chainstate-dir
95+
@echo "Logs for service $(PARAMS)"
96+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) logs -t --no-log-prefix $(PARAMS) -f
97+
98+
# Stream all services logs to STDOUT
99+
log-all: current-chainstate-dir
100+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) logs -t -f
101+
102+
# Backup all service logs to $ACTIVE_CHAINSTATE_DIR/logs/<service-name>.log
103+
backup-logs: current-chainstate-dir /usr/bin/sudo
104+
@if [ -f .current-chainstate-dir ]; then \
105+
$(eval ACTIVE_CHAINSTATE_DIR=$(shell cat .current-chainstate-dir))
106+
if [ ! -d "$(ACTIVE_CHAINSTATE_DIR)" ]; then \
107+
echo "Chainstate Dir ($(ACTIVE_CHAINSTATE_DIR)) not found";\
108+
exit 1; \
109+
fi; \
110+
if [ ! -d "$(ACTIVE_CHAINSTATE_DIR)/logs" ]; then \
111+
mkdir -p $(ACTIVE_CHAINSTATE_DIR)/logs;\
112+
fi; \
113+
echo "Backing up logs to $(ACTIVE_CHAINSTATE_DIR)/logs"; \
114+
for service in $(SERVICES); do \
115+
docker logs -t $$service > $(ACTIVE_CHAINSTATE_DIR)/logs/$$service.log 2>&1; \
116+
done; \
117+
fi
118+
119+
# set env var of what the statically defined chainstate dir is
120+
current-chainstate-dir: | check-running
121+
$(eval ACTIVE_CHAINSTATE_DIR=$(shell cat .current-chainstate-dir))
37122

38-
log:
39-
docker compose -f docker/docker-compose.yml --profile=default logs -t --no-log-prefix "$(Arguments)" -f
123+
# replace the existing chainstate archive. will be used with target `up`
124+
snapshot: current-chainstate-dir down
125+
@echo "Creating $(PROJECT) chainstate snapshot from $(ACTIVE_CHAINSTATE_DIR)"
126+
@if [ -d "$(ACTIVE_CHAINSTATE_DIR)/logs" ]; then \
127+
rm -rf $(ACTIVE_CHAINSTATE_DIR)/logs; \
128+
fi
129+
@echo "Creating snapshot: $(CHAINSTATE_ARCHIVE)"
130+
@echo "cd $(ACTIVE_CHAINSTATE_DIR); sudo tar --zstd -cf $(CHAINSTATE_ARCHIVE) *; cd $(PWD)"
131+
cd $(ACTIVE_CHAINSTATE_DIR); sudo tar --zstd -cf $(CHAINSTATE_ARCHIVE) *; cd $(PWD)
132+
133+
# pause all services in the network (netork is down, but recoverably with target 'unpause')
134+
pause:
135+
@echo "Pausing all services"
136+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) pause $(SERVICES)
137+
138+
# unpause all services in the network (only used after first using target 'pause')
139+
unpause:
140+
@echo "Unpausing all services"
141+
docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) unpause $(SERVICES)
142+
143+
# stop an individual service
144+
stop: check-params current-chainstate-dir | check-running
145+
@echo "Killing service $(PARAMS)"
146+
@echo " Chainstate Dir: $(ACTIVE_CHAINSTATE_DIR)"
147+
@echo " Target: $(TARGET)"
148+
@echo " Params: $(PARAMS)"
149+
CHAINSTATE_DIR=$(ACTIVE_CHAINSTATE_DIR) docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) down $(PARAMS)
150+
151+
# start an individual service
152+
start: check-params current-chainstate-dir | check-running
153+
@echo "Resuming service $(PARAMS)"
154+
@echo " Chainstate Dir: $(ACTIVE_CHAINSTATE_DIR)"
155+
@echo " Target: $(TARGET)"
156+
@echo " Params: $(PARAMS)"
157+
CHAINSTATE_DIR=$(ACTIVE_CHAINSTATE_DIR) docker compose -f docker/docker-compose.yml --profile default -p $(PROJECT) up -d $(PARAMS)
158+
159+
# restart a service with a defined servicename/duration. called script will validate PARAMS
160+
# if no duration is provided, default of 30s shall be used
161+
restart: check-params | check-running
162+
@echo "Restarting service"
163+
@echo " Params: $(PARAMS)"
164+
./docker/tests/restart-container.sh $(PARAMS)
165+
166+
# use 'stress' binary to consume defined cpu over a specified time
167+
stress:
168+
@echo "Stressing system CPU $(PARAMS)"
169+
@echo " Cores: $(STRESS_CORES)"
170+
@echo " Timeout: $(STRESS_TIMEOUT)"
171+
stress --cpu $(STRESS_CORES) --timeout $(STRESS_TIMEOUT)
172+
173+
# run the test script to verify the services are all load and operating as expected
174+
test:
175+
./docker/tests/devnet-liveness.sh
176+
177+
# run the chain monitor script (loops and curls /v2/info, parsing the output to show current heights of miners)
178+
monitor:
179+
./docker/tests/chain-monitor.sh
180+
181+
# if the network is already running, we need to exit (ex: trying to start the network when it's already running)
182+
check-not-running:
183+
@if test `docker compose ls --filter name=$(PROJECT) -q`; then \
184+
echo ""; \
185+
echo "WARNING: Network appears to be running or was not properly shut down."; \
186+
echo "Current chainstate directory: $$(cat .current-chainstate-dir)"; \
187+
echo ""; \
188+
echo "To backup logs first: make backup-logs"; \
189+
echo "To shut down: make down"; \
190+
echo ""; \
191+
exit 1; \
192+
fi
193+
194+
# if the network is not running, we need to exit (ex: trying to restart a container)
195+
check-running:
196+
@if test ! `docker compose ls --filter name=$(PROJECT) -q`; then \
197+
echo "Network not running. exiting"; \
198+
exit 1; \
199+
fi
200+
201+
# for targets that need an arg, check that *something* is provided. it not, exit
202+
check-params: | check-running
203+
@if [ ! "$(PARAMS)" ]; then \
204+
echo "No service defined. Exiting"; \
205+
exit 1; \
206+
fi
40207

41-
log-all:
42-
docker compose -f docker/docker-compose.yml --profile=default logs -t -f
208+
# force stop and remove any existing chainstates (leaving all docker images/layers)
209+
clean: down-force
210+
sudo rm -rf ./docker/chainstate/*
43211

44-
build:
45-
COMPOSE_BAKE=true PWD=$(PWD) docker compose -f docker/docker-compose.yml --profile default build
46212

47-
.PHONY: up down up-genesis down-genesis log log-all
213+
.PHONY: up genesis up-genesis down-genesis down down-force build build-no-cache log log-all backup-logs current-chainstate-dir snapshot pause unpause stop start restart stress test monitor check-not-running check-running check-params clean
48214
.ONESHELL: all-in-one-shell

0 commit comments

Comments
 (0)