Created
December 27, 2023 14:24
-
-
Save Slach/b23b8d125ee0c6e7d48337522a4d9057 to your computer and use it in GitHub Desktop.
reproduce ALTER TABLE FREEZE for encrypted over s3 disk for ReplicatedMergeTree
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<clickhouse> | |
<listen_host replace="replace">0.0.0.0</listen_host> | |
<logger> | |
<console>1</console> | |
</logger> | |
<keeper_server> | |
<tcp_port>2181</tcp_port> | |
</keeper_server> | |
</clickhouse> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
services: | |
minio: | |
image: docker.io/bitnami/minio:${MINIO_VERSION:-latest} | |
environment: | |
MINIO_ACCESS_KEY: access-key | |
MINIO_SECRET_KEY: it-is-my-super-secret-key | |
MINIO_DEFAULT_BUCKETS: 'clickhouse' | |
MINIO_ROOT_USER: access-key | |
MINIO_ROOT_PASSWORD: it-is-my-super-secret-key | |
healthcheck: | |
test: curl -sL http://localhost:9000/ | |
interval: 10s | |
retries: 30 | |
keeper: | |
image: ${CLICKHOUSE_KEEPER_IMAGE:-docker.io/clickhouse/clickhouse-keeper}:${CLICKHOUSE_KEEPER_VERSION:-latest-alpine} | |
volumes: | |
- ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml | |
healthcheck: | |
test: bash -c 'if [[ "$$(echo 'ruok' | nc 127.0.0.1 2181)" == "imok" ]]; then exit 0; else exit 1; fi' | |
interval: 3s | |
timeout: 2s | |
retries: 5 | |
start_period: 2s | |
clickhouse: | |
image: ${CLICKHOUSE_SERVER_IMAGE:-docker.io/clickhouse/clickhouse-server}:${CLICKHOUSE_SERVER_VERSION:-latest-alpine} | |
volumes: | |
- ./storage_configuration.xml:/etc/clickhouse-server/config.d/storage_configuration.xml | |
healthcheck: | |
test: clickhouse client -q "SELECT 1" | |
interval: 10s | |
timeout: 2s | |
retries: 30 | |
start_period: 5s | |
links: | |
- keeper | |
- minio | |
depends_on: | |
minio: | |
condition: service_healthy | |
keeper: | |
condition: service_healthy |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -xeo pipefail | |
docker-compose down | |
docker-compose up -d clickhouse | |
docker-compose exec clickhouse clickhouse-client -q "CREATE DATABASE test" | |
docker-compose exec clickhouse clickhouse-client -q "CREATE TABLE test.test (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY tuple() SETTINGS storage_policy='s3_only'" | |
docker-compose exec clickhouse clickhouse-client -q "INSERT INTO TABLE test.test SELECT number FROM numbers(1000000)" | |
docker-compose exec clickhouse clickhouse-client -q "ALTER TABLE test.test FREEZE WITH NAME 'test'" | |
docker-compose exec clickhouse clickhouse-client -q "DROP TABLE test.test SYNC" | |
docker-compose exec clickhouse clickhouse-client -q "CREATE TABLE test.test_encrypted (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY tuple() SETTINGS storage_policy='s3_only_encrypted'" | |
docker-compose exec clickhouse clickhouse-client -q "INSERT INTO TABLE test.test_encrypted SELECT number FROM numbers(1000000)" | |
docker-compose exec clickhouse clickhouse-client -q "ALTER TABLE test.test_encrypted FREEZE WITH NAME 'test'" | |
docker-compose exec clickhouse clickhouse-client -q "DROP TABLE test.test_encrypted SYNC" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<clickhouse> | |
<storage_configuration> | |
<disks> | |
<disk_s3> | |
<type>s3</type> | |
<endpoint>http://minio:9000/clickhouse/disk_s3/</endpoint> | |
<access_key_id>access-key</access_key_id> | |
<secret_access_key>it-is-my-super-secret-key</secret_access_key> | |
<!-- to avoid slow startup --> | |
<send_metadata>false</send_metadata> | |
</disk_s3> | |
<disk_s3_encrypted> | |
<type>encrypted</type> | |
<disk>disk_s3</disk> | |
<path>disk_s3_encrypted/</path> | |
<algorithm>AES_128_CTR</algorithm> | |
<key_hex id="0">00112233445566778899aabbccddeeff</key_hex> | |
<key_hex id="1">ffeeddccbbaa99887766554433221100</key_hex> | |
<current_key_id>1</current_key_id> | |
<!-- to avoid slow startup --> | |
<send_metadata>false</send_metadata> | |
</disk_s3_encrypted> | |
</disks> | |
<policies> | |
<s3_only> | |
<volumes> | |
<s3_only> | |
<disk>disk_s3</disk> | |
</s3_only> | |
</volumes> | |
</s3_only> | |
<s3_only_encrypted> | |
<volumes> | |
<s3_only_encrypted> | |
<disk>disk_s3_encrypted</disk> | |
</s3_only_encrypted> | |
</volumes> | |
</s3_only_encrypted> | |
</policies> | |
</storage_configuration> | |
<macros> | |
<cluster>cluster</cluster> | |
<shard>0</shard> | |
<replica>clickhouse</replica> | |
</macros> | |
<zookeeper> | |
<node index="1"> | |
<host>keeper</host> | |
<port>2181</port> | |
</node> | |
</zookeeper> | |
</clickhouse> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment