Compare commits
6 Commits
695d095133
...
a9c1d87b38
| Author | SHA1 | Date |
|---|---|---|
 Федоров Дмитрий
|
a9c1d87b38 | |
|
Федоров Дмитрий
|
3310c3143e | |
|
Федоров Дмитрий
|
91e73ffbef | |
|
Федоров Дмитрий
|
3f10edf636 | |
|
Федоров Дмитрий
|
5d7f0eb094 | |
|
Федоров Дмитрий
|
6864921ba2 |
|
|
@ -0,0 +1,164 @@
|
|||
---
|
||||
title: 0. Основы и запросы
|
||||
description:
|
||||
published: true
|
||||
date: 2025-05-15T20:02:37.600Z
|
||||
tags: clickhouse
|
||||
editor: markdown
|
||||
dateCreated: 2025-05-15T20:01:45.285Z
|
||||
---
|
||||
|
||||
## Структура кластера
|
||||
|
||||
### Найдено на просторах инета:
|
||||
##### Выбор решения:
|
||||
- Если **диск загружен на запись (INSERT)** → **шардирование (Distributed)**.
|
||||
- Если **диск загружен на чтение (SELECT)** → **репликация (Replicated)**.
|
||||
- Если **один сервер, но много дисков** → **JBOD или RAID 0/10**.
|
||||
|
||||
#### Реализуем такую структуру тестового кластера:
|
||||
- сервер clickhouse-1
|
||||
- сервер clickhouse-2
|
||||
- сервер clickhouse-keeper
|
||||
Для того, чтобы это было надежно и работало быстро с записью в БД, необходимо сделать репликацию (Replicated) таблиц и распределенный (Distributed) доступ к таблицам.
|
||||
Если делать несколько шардов (shard) в настройках кластера, то это значит, что часть данных будет находится на одном сервере, другая часть на другом. При выходе из строя сервера мы потеряем всю базу.
|
||||
Поэтому настраиваем shard1 на обоих серверах, а также replica1 и replica2 для дублирования данных на серверах.
|
||||
|
||||
### Репликация поддерживается только для таблиц семейства MergeTree:
|
||||
https://clickhouse.com/docs/ru/engines/table-engines/mergetree-family/replication
|
||||
- ReplicatedMergeTree
|
||||
- ReplicatedSummingMergeTree
|
||||
- ReplicatedReplacingMergeTree
|
||||
- ReplicatedAggregatingMergeTree
|
||||
- ReplicatedCollapsingMergeTree
|
||||
- ReplicatedVersionedCollapsingMergeTree
|
||||
- ReplicatedGraphiteMergeTree
|
||||
|
||||
### Для одиночного сервера и кластера
|
||||
#### Получение списка всех баз данных на сервере
|
||||
```sql
|
||||
SHOW DATABASES;
|
||||
```
|
||||
#### Просмотр всех таблиц в конкретной базе default
|
||||
```sql
|
||||
SHOW TABLES FROM default;
|
||||
```
|
||||
#### Получить запрос на создание таблицы из существующей
|
||||
```sql
|
||||
show create table default.event;
|
||||
```
|
||||
#### Создание базы данных
|
||||
```sql
|
||||
CREATE DATABASE IF NOT EXISTS my_database ENGINE = Atomic;
|
||||
```
|
||||
Создает новую базу данных с именем `my_database`. Если такая база уже существует, ничего не произойдет благодаря условию `IF NOT EXISTS`.
|
||||
Параметр `ENGINE = Atomic` используется для новых версий ClickHouse и позволяет создавать транзакционно безопасные базы данных. В ранних версиях использовался простой синтаксис без указания движка.
|
||||
#### Создание таблицы
|
||||
```sql
|
||||
CREATE TABLE my_table (
|
||||
event_date Date,
|
||||
user_id UInt32,
|
||||
page_views Int32,
|
||||
revenue Float64
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (event_date);
|
||||
```
|
||||
##### Вставка данных
|
||||
```sql
|
||||
INSERT INTO my_table VALUES ('2023-08-01', 1, 10, 100.5), ('2023-08-02', 2, 15, 150.75);
|
||||
```
|
||||
##### Выборка данных
|
||||
```sql
|
||||
SELECT * FROM my_table WHERE event_date >= '2023-08-01' AND event_date <= '2023-08-31';
|
||||
```
|
||||
#### Показ структуры таблицы
|
||||
```sql
|
||||
DESCRIBE TABLE my_database.my_table;
|
||||
```
|
||||
или подробнее:
|
||||
```sql
|
||||
SELECT * FROM system.columns WHERE table='my_table' AND database='my_database';
|
||||
```
|
||||
### Узнать сколько занимают места базы
|
||||
```sql
|
||||
SELECT
|
||||
database,
|
||||
formatReadableSize(sum(bytes_on_disk)) AS total_size
|
||||
FROM system.parts
|
||||
WHERE active
|
||||
GROUP BY database
|
||||
ORDER BY sum(bytes_on_disk) DESC;
|
||||
```
|
||||
#### Узнать сколько занимают места таблицы в базе
|
||||
default
|
||||
```sql
|
||||
SELECT
|
||||
name AS table_name,
|
||||
formatReadableSize(total_bytes) AS size_readable,
|
||||
total_rows AS rows_count
|
||||
FROM system.tables
|
||||
WHERE database = 'default'
|
||||
ORDER BY total_bytes DESC;
|
||||
```
|
||||
system
|
||||
```sql
|
||||
SELECT
|
||||
name AS table_name,
|
||||
formatReadableSize(total_bytes) AS size_readable,
|
||||
total_rows AS rows_count
|
||||
FROM system.tables
|
||||
WHERE database = 'system'
|
||||
ORDER BY total_bytes DESC;
|
||||
```
|
||||
#### Очистка базы event_replicated
|
||||
```sql
|
||||
TRUNCATE TABLE default.event_replicated;
|
||||
```
|
||||
#### Очистка старых данных от определенной даты
|
||||
```sql
|
||||
ALTER TABLE my_table DELETE WHERE event_date < '2023-08-01';
|
||||
```
|
||||
#### Безопасное удаление таблицы
|
||||
```sql
|
||||
DROP TABLE IF EXISTS my_database.my_table;
|
||||
```
|
||||
Безопасно удаляет указанную таблицу, если она существует.
|
||||
|
||||
|
||||
### Для кластера
|
||||
|
||||
#### Вывести список серверов в кластере
|
||||
```sql
|
||||
SELECT hostName() FROM clusterAllReplicas('smvu2_cluster', 'system', 'one')
|
||||
```
|
||||
#### Получение списка Replicated таблиц
|
||||
```sql
|
||||
SELECT
|
||||
database,
|
||||
name AS table,
|
||||
engine
|
||||
FROM system.tables
|
||||
WHERE engine LIKE 'Replicated%';
|
||||
```
|
||||
#### Получение списка Distributed таблиц
|
||||
```sql
|
||||
SELECT
|
||||
database,
|
||||
name AS table,
|
||||
engine
|
||||
FROM system.tables
|
||||
WHERE engine LIKE 'Distributed%';
|
||||
```
|
||||
#### Проверить распределение данных базы event на серверах кластера
|
||||
```sql
|
||||
SELECT
|
||||
hostName() AS host,
|
||||
count() AS count
|
||||
FROM clusterAllReplicas(smvu2_cluster, default.event)
|
||||
GROUP BY host;
|
||||
```
|
||||
#### Удалить таблицу в кластере
|
||||
```sql
|
||||
DROP TABLE event_replicated ON CLUSTER smvu2_cluster SYNC;
|
||||
```
|
||||
|
|
@ -2,14 +2,14 @@
|
|||
title: 1. Создание кластера Clickhouse
|
||||
description:
|
||||
published: true
|
||||
date: 2025-05-15T18:04:57.423Z
|
||||
date: 2025-05-15T20:11:12.215Z
|
||||
tags: clickhouse, cluster
|
||||
editor: markdown
|
||||
dateCreated: 2025-05-15T18:04:57.423Z
|
||||
---
|
||||
|
||||
https://clickhouse.com/docs/guides/sre/keeper/clickhouse-keeper
|
||||
|
||||
## Установка и настройка clickhouse, clickhouse-keeper
|
||||
### Установка clickhouse-keeper
|
||||
```bash
|
||||
apt-get install -y apt-transport-https ca-certificates curl gnupg
|
||||
|
|
@ -20,7 +20,7 @@ apt-get update
|
|||
apt-get install -y clickhouse-keeper
|
||||
```
|
||||
|
||||
### Готовим конфиг clickhouse-keeper
|
||||
#### Готовим конфиг clickhouse-keeper
|
||||
```
|
||||
nano /etc/clickhouse-keeper/keeper_config.xml
|
||||
```
|
||||
|
|
@ -67,7 +67,7 @@ nano /etc/clickhouse-keeper/keeper_config.xml
|
|||
</clickhouse>
|
||||
```
|
||||
|
||||
### Пароль пользователя
|
||||
#### Пароль пользователя
|
||||
```
|
||||
nano /etc/clickhouse-server/users.d/default-password.xml
|
||||
```
|
||||
|
|
@ -92,7 +92,7 @@ nano /etc/clickhouse-server/users.d/default-password.xml
|
|||
```bash
|
||||
nano /etc/clickhouse-server/config.d/remote_servers.xml
|
||||
```
|
||||
#### server 1
|
||||
##### server 1
|
||||
```xml
|
||||
<yandex>
|
||||
<display_name>smvu2_cluster node 1</display_name>
|
||||
|
|
@ -136,7 +136,7 @@ nano /etc/clickhouse-server/config.d/shard.xml
|
|||
</yandex>
|
||||
```
|
||||
|
||||
#### server 2
|
||||
##### server 2
|
||||
```xml
|
||||
<yandex>
|
||||
<display_name>smvu2_cluster node 2</display_name>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,646 @@
|
|||
---
|
||||
title: 3. Создание таблиц вручную
|
||||
description:
|
||||
published: true
|
||||
date: 2025-05-15T20:07:34.738Z
|
||||
tags: clickhouse
|
||||
editor: markdown
|
||||
dateCreated: 2025-05-15T20:07:34.738Z
|
||||
---
|
||||
|
||||
Можно создать все таблицы для кластера вручную.
|
||||
Запрашиваем структуру каждой таблицы, создаем под другим именем и копируем данные с исходника.
|
||||
|
||||
##### Может возникнуть проблема, что таблицы содержат много данных, поэтому внизу статьи будет решение
|
||||
|
||||
## Создаем реплицированные таблицы
|
||||
|
||||
**Разберем первую таблицу подробно**
|
||||
### 1. api_request_log_replicated
|
||||
|
||||
Запрашиваем структуру
|
||||
```sql
|
||||
show create table default.api_request_log
|
||||
```
|
||||
|
||||
Выход:
|
||||
```sql
|
||||
CREATE TABLE default.api_request_log
|
||||
(
|
||||
`uuid` UUID DEFAULT '00000000-0000-0000-0000-000000000000',
|
||||
`ver` UInt8,
|
||||
`request_ts` DateTime64(3) DEFAULT '0000000000.000',
|
||||
`duration` Float64,
|
||||
`endpoint` String,
|
||||
`method` FixedString(5),
|
||||
`status_code` UInt32,
|
||||
`remote_addr` String,
|
||||
`username` Nullable(String),
|
||||
`error` Nullable(String),
|
||||
`request_body` Nullable(String) TTL toDateTime(request_ts) + toIntervalMonth(1),
|
||||
`query_params` Nullable(String) TTL toDateTime(request_ts) + toIntervalMonth(1),
|
||||
`response_body` Nullable(String) TTL toDateTime(request_ts) + toIntervalMonth(1)
|
||||
)
|
||||
ENGINE = MergeTree(ver)
|
||||
PARTITION BY toYYYYMM(request_ts)
|
||||
ORDER BY uuid
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
|
||||
Дорабатываем запрос, чтобы получилась на выходе реплицируемая таблица
|
||||
```sql
|
||||
CREATE TABLE default.api_request_log_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`uuid` UUID DEFAULT '00000000-0000-0000-0000-000000000000',
|
||||
`ver` UInt8,
|
||||
`request_ts` DateTime64(3) DEFAULT '0000000000.000',
|
||||
`duration` Float64,
|
||||
`endpoint` String,
|
||||
`method` FixedString(5),
|
||||
`status_code` UInt32,
|
||||
`remote_addr` String,
|
||||
`username` Nullable(String),
|
||||
`error` Nullable(String),
|
||||
`request_body` Nullable(String) TTL toDateTime(request_ts) + toIntervalMonth(1),
|
||||
`query_params` Nullable(String) TTL toDateTime(request_ts) + toIntervalMonth(1),
|
||||
`response_body` Nullable(String) TTL toDateTime(request_ts) + toIntervalMonth(1)
|
||||
)
|
||||
ENGINE = ReplicatedReplacingMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/api_request_log_replicated',
|
||||
'{replica}',
|
||||
ver
|
||||
)
|
||||
PARTITION BY toYYYYMM(request_ts)
|
||||
ORDER BY uuid
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем данные
|
||||
```sql
|
||||
INSERT INTO default.api_request_log_replicated
|
||||
SELECT * FROM default.api_request_log
|
||||
```
|
||||
|
||||
### 2. event_replicated
|
||||
```sql
|
||||
show create table default.event
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`event_id` UUID,
|
||||
`unipoll_id` UUID,
|
||||
`sensor_id` UUID,
|
||||
`sensor_type_id` UUID,
|
||||
`sensor_type_name` String,
|
||||
`sensor_activation_status_name` String,
|
||||
`sensor_logic_status_id` UUID,
|
||||
`sensor_logic_status_name` String,
|
||||
`sensor_comment` Nullable(String),
|
||||
`command` String,
|
||||
`priority_name` String,
|
||||
`pub_ts` DateTime,
|
||||
`sub_ts` DateTime64(3, 'Europe/Moscow'),
|
||||
`state_id` UInt8,
|
||||
`state_name` String,
|
||||
`status_id` UInt8,
|
||||
`status_name` String,
|
||||
`unitype_id` UInt16,
|
||||
`unitype_name` String,
|
||||
`system_id` UInt16,
|
||||
`system_short_name` String,
|
||||
`system_name` String,
|
||||
`checked` UInt8,
|
||||
`object_id` UUID,
|
||||
`object_type_name` String,
|
||||
`data_id` UInt8,
|
||||
`value` Nullable(String),
|
||||
`raw_value` Nullable(String),
|
||||
`data` String,
|
||||
`region_id` UUID,
|
||||
`region_name` String,
|
||||
`complex_id` UUID,
|
||||
`complex_name` String,
|
||||
`collector_id` Nullable(UUID),
|
||||
`collector_name` Nullable(String),
|
||||
`control_room_id` Nullable(UUID),
|
||||
`control_room_name` Nullable(String),
|
||||
`picket_id` Nullable(UUID),
|
||||
`picket_name` Nullable(String),
|
||||
`picket_n` Nullable(UInt32),
|
||||
`cstate_id` UUID,
|
||||
`cstate_name` String,
|
||||
`cstate_color_id` UInt8,
|
||||
`sub_ms` UInt64,
|
||||
`linked_channel_state` Nullable(String),
|
||||
`on_guard` Nullable(UInt8),
|
||||
`internal_id` Nullable(String),
|
||||
`floor_number` Nullable(Int32)
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/event_replicated',
|
||||
'{replica}'
|
||||
)
|
||||
PARTITION BY toYYYYMM(pub_ts)
|
||||
ORDER BY (priority_name, sensor_type_id, cstate_id, object_id, sensor_id, pub_ts, sub_ts, event_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем данные
|
||||
```sql
|
||||
INSERT INTO default.event_replicated
|
||||
SELECT * FROM default.event
|
||||
```
|
||||
|
||||
### 3. event_automation_replicated
|
||||
```sql
|
||||
show create table default.event_automation
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event_automation_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`event_id` UUID DEFAULT '00000000-0000-0000-0000-000000000000',
|
||||
`unipoll_id` UUID DEFAULT '00000000-0000-0000-0000-000000000000',
|
||||
`automation_id` UUID DEFAULT '00000000-0000-0000-0000-000000000000',
|
||||
`automation_name` String,
|
||||
`automation_mode` UInt8,
|
||||
`pub_ts` DateTime DEFAULT '0000000000',
|
||||
`sub_ts` DateTime64(3) DEFAULT '0000000000.000',
|
||||
`sub_ms` UInt64,
|
||||
`state_id` UInt8,
|
||||
`state_name` String,
|
||||
`status_id` UInt8,
|
||||
`unitype_id` UInt8,
|
||||
`unitype_name` String,
|
||||
`group_id` UUID DEFAULT '00000000-0000-0000-0000-000000000000',
|
||||
`group_name` String,
|
||||
`data` String,
|
||||
`cstate_id` UUID DEFAULT '00000000-0000-0000-0000-000000000000',
|
||||
`cstate_name` String,
|
||||
`cstate_color_id` UInt8
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/event_automation_replicated',
|
||||
'{replica}'
|
||||
)
|
||||
PARTITION BY toYYYYMMDD(pub_ts)
|
||||
ORDER BY (event_id, pub_ts, sub_ts)
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем данные
|
||||
```sql
|
||||
INSERT INTO default.event_automation_replicated
|
||||
SELECT * FROM default.event_automation
|
||||
```
|
||||
|
||||
### 4. event_confirmation_replicated
|
||||
```sql
|
||||
show create table default.event_confirmation
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event_confirmation_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`uuid` UUID,
|
||||
`event_id` UUID,
|
||||
`reason` String DEFAULT '',
|
||||
`event_checked` Int8,
|
||||
`username` String,
|
||||
`datetime` DateTime,
|
||||
`confirmation_group` String,
|
||||
`node_id` UUID
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/event_confirmation_replicated',
|
||||
'{replica}'
|
||||
)
|
||||
PARTITION BY toYYYYMM(datetime)
|
||||
ORDER BY datetime
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем данные
|
||||
```sql
|
||||
INSERT INTO default.event_confirmation_replicated
|
||||
SELECT * FROM default.event_confirmation
|
||||
```
|
||||
|
||||
### 5. event_objects_replicated
|
||||
```sql
|
||||
show create table default.event_objects
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event_objects_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`event_id` UUID,
|
||||
`node_id` UUID,
|
||||
`pub_ts` DateTime,
|
||||
`sub_ts` DateTime64(3, 'UTC'),
|
||||
`cstate_id` UUID,
|
||||
`source_id` UUID,
|
||||
`source_name` String DEFAULT '',
|
||||
`source_type_id` UUID,
|
||||
`event_data` String,
|
||||
`complex_id` UUID
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/event_objects_replicated',
|
||||
'{replica}'
|
||||
)
|
||||
PARTITION BY toYYYYMM(pub_ts)
|
||||
PRIMARY KEY event_id
|
||||
ORDER BY (event_id, pub_ts, sub_ts)
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем данные
|
||||
```sql
|
||||
INSERT INTO default.event_objects_replicated
|
||||
SELECT * FROM default.event_objects
|
||||
```
|
||||
### 6. infi_clickhouse_orm_migrations_replicated
|
||||
|
||||
```sql
|
||||
show create table default.infi_clickhouse_orm_migrations
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.infi_clickhouse_orm_migrations_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`package_name` String,
|
||||
`module_name` String,
|
||||
`applied` Date DEFAULT '1970-01-01'
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/infi_clickhouse_orm_migrations_replicated',
|
||||
'{replica}'
|
||||
)
|
||||
PARTITION BY toYYYYMM(applied)
|
||||
ORDER BY (package_name, module_name)
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем данные
|
||||
```sql
|
||||
INSERT INTO default.infi_clickhouse_orm_migrations_replicated
|
||||
SELECT * FROM default.infi_clickhouse_orm_migrations
|
||||
```
|
||||
|
||||
### 7. unchecked_event_replicated
|
||||
|
||||
```sql
|
||||
show create table default.unchecked_event
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.unchecked_event_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`event_id` UUID,
|
||||
`unipoll_id` Nullable(UUID),
|
||||
`sensor_id` Nullable(UUID),
|
||||
`sensor_type_id` Nullable(UUID),
|
||||
`sensor_type_name` Nullable(String),
|
||||
`sensor_activation_status_name` Nullable(String),
|
||||
`sensor_logic_status_id` Nullable(UUID),
|
||||
`sensor_logic_status_name` Nullable(String),
|
||||
`sensor_comment` Nullable(String),
|
||||
`command` Nullable(String),
|
||||
`priority_name` Nullable(String),
|
||||
`pub_ts` Nullable(DateTime),
|
||||
`sub_ts` DateTime64(3),
|
||||
`state_id` Nullable(UInt8),
|
||||
`state_name` Nullable(String),
|
||||
`status_id` Nullable(UInt8),
|
||||
`status_name` Nullable(String),
|
||||
`unitype_id` Nullable(UInt16),
|
||||
`unitype_name` Nullable(String),
|
||||
`system_id` Nullable(UInt16),
|
||||
`system_short_name` Nullable(String),
|
||||
`system_name` Nullable(String),
|
||||
`checked` Nullable(UInt8),
|
||||
`object_id` Nullable(UUID),
|
||||
`object_type_name` Nullable(String),
|
||||
`data_id` Nullable(UInt8),
|
||||
`data` Nullable(String),
|
||||
`region_id` Nullable(UUID),
|
||||
`region_name` Nullable(String),
|
||||
`complex_id` Nullable(UUID),
|
||||
`complex_name` Nullable(String),
|
||||
`collector_id` Nullable(UUID),
|
||||
`collector_name` Nullable(String),
|
||||
`control_room_id` Nullable(UUID),
|
||||
`control_room_name` Nullable(String),
|
||||
`picket_id` Nullable(UUID),
|
||||
`picket_name` Nullable(String),
|
||||
`picket_n` Nullable(UInt32),
|
||||
`cstate_id` Nullable(UUID),
|
||||
`cstate_name` Nullable(String),
|
||||
`cstate_color_id` Nullable(UInt8),
|
||||
`sub_ms` Nullable(UInt64),
|
||||
`linked_channel_state` Nullable(String),
|
||||
`on_guard` Nullable(UInt8),
|
||||
`internal_id` Nullable(String),
|
||||
`floor_number` Nullable(Int32),
|
||||
`sign` Int8,
|
||||
`value` Nullable(String),
|
||||
`raw_value` Nullable(String)
|
||||
)
|
||||
ENGINE = ReplicatedCollapsingMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/unchecked_event_replicated',
|
||||
'{replica}',
|
||||
sign
|
||||
)
|
||||
PARTITION BY toYYYYMM(sub_ts)
|
||||
PRIMARY KEY event_id
|
||||
ORDER BY event_id
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем
|
||||
```sql
|
||||
INSERT INTO default.unchecked_event_replicated
|
||||
SELECT * FROM default.unchecked_event;
|
||||
```
|
||||
|
||||
### 8. unchecked_objects_event_replicated
|
||||
```sql
|
||||
show create table default.unchecked_objects_event;
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.unchecked_objects_event_replicated ON CLUSTER smvu2_cluster
|
||||
(
|
||||
`event_id` UUID,
|
||||
`node_id` Nullable(UUID),
|
||||
`complex_id` Nullable(UUID),
|
||||
`pub_ts` Nullable(DateTime),
|
||||
`sub_ts` DateTime64(3, 'UTC'),
|
||||
`cstate_id` Nullable(UUID),
|
||||
`source_id` Nullable(UUID),
|
||||
`source_name` Nullable(String),
|
||||
`source_type_id` Nullable(UUID),
|
||||
`event_data` Nullable(String),
|
||||
`sign` Int8
|
||||
)
|
||||
ENGINE = ReplicatedCollapsingMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/unchecked_objects_event_replicated',
|
||||
'{replica}',
|
||||
sign
|
||||
)
|
||||
PARTITION BY toYYYYMM(sub_ts)
|
||||
PRIMARY KEY event_id
|
||||
ORDER BY event_id
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
Копируем
|
||||
```sql
|
||||
INSERT INTO default.unchecked_objects_event_replicated
|
||||
SELECT * FROM default.unchecked_objects_event;
|
||||
```
|
||||
|
||||
|
||||
### Проверка состояния репликации после создания:
|
||||
```sql
|
||||
SELECT database, table, is_leader, total_replicas, active_replicas
|
||||
FROM system.replicas
|
||||
WHERE database = 'default'
|
||||
```
|
||||
### Узнаем и сравниваем сколько занимают места таблицы в базе default
|
||||
```sql
|
||||
SELECT
|
||||
name AS table_name,
|
||||
formatReadableSize(total_bytes) AS size_readable,
|
||||
total_rows AS rows_count
|
||||
FROM system.tables
|
||||
WHERE database = 'default'
|
||||
ORDER BY total_bytes DESC;
|
||||
```
|
||||
|
||||
### Переименовываем исходные таблицы
|
||||
```sql
|
||||
rename table api_request_log to api_request_log_old;
|
||||
rename table event to event_old;
|
||||
rename table event_objects to event_objects_old;
|
||||
rename table event_confirmation to event_confirmation_old;
|
||||
rename table audit_new to audit_new_old;
|
||||
rename table event_automation to event_automation_old;
|
||||
rename table infi_clickhouse_orm_migrations to infi_clickhouse_orm_migrations_old;
|
||||
rename table unchecked_event to unchecked_event_old;
|
||||
rename table unchecked_objects_event to unchecked_objects_event_old;
|
||||
```
|
||||
|
||||
### Создаем Distributed-таблицы
|
||||
```sql
|
||||
CREATE TABLE default.event ON CLUSTER smvu2_cluster AS default.event_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, api_request_log_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event ON CLUSTER smvu2_cluster AS default.event_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, event_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.audit_new ON CLUSTER smvu2_cluster AS default.audit_new_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, audit_new_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event_objects ON CLUSTER smvu2_cluster AS default.event_objects_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, event_objects_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event_automation ON CLUSTER smvu2_cluster AS default.event_automation_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, event_automation_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.event_confirmation ON CLUSTER smvu2_cluster AS default.event_confirmation_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, event_confirmation_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.infi_clickhouse_orm_migrations ON CLUSTER smvu2_cluster AS default.infi_clickhouse_orm_migrations_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, infi_clickhouse_orm_migrations_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.unchecked_event ON CLUSTER smvu2_cluster AS default.unchecked_event_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, unchecked_event_replicated, rand());
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE default.unchecked_objects_event ON CLUSTER smvu2_cluster AS default.unchecked_objects_event_replicated
|
||||
ENGINE = Distributed(smvu2_cluster, default, unchecked_objects_event_replicated, rand());
|
||||
```
|
||||
|
||||
|
||||
### Решение проблемы размера таблицы
|
||||
#### 1. Используя ограничение отправляемых строк, прикладываю скрипт, который делает в автоматическом режиме
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
|
||||
# Список таблиц для репликации
|
||||
TABLES=(
|
||||
api_request_log
|
||||
audit_new
|
||||
event
|
||||
event_automation
|
||||
event_confirmation
|
||||
event_objects
|
||||
infi_clickhouse_orm_migrations
|
||||
)
|
||||
|
||||
PASSWORD="q1"
|
||||
BATCH_SIZE=100000 # Размер порции для вставки
|
||||
MAX_THREADS=4 # Максимальное количество потоков для вставки
|
||||
|
||||
for table in "${TABLES[@]}"; do
|
||||
replicated_table="${table}_replicated"
|
||||
echo "Processing $table to $replicated_table"
|
||||
|
||||
# 1. Получаем общее количество строк
|
||||
total_rows=$(clickhouse-client --password "$PASSWORD" --query="
|
||||
SELECT count() FROM remote('localhost', 'default', '$table')")
|
||||
|
||||
echo "Total rows to copy: $total_rows"
|
||||
|
||||
# 2. Копируем данные порциями
|
||||
for ((offset=0; offset<total_rows; offset+=BATCH_SIZE)); do
|
||||
echo "Copying rows $((offset+1))-$((offset+BATCH_SIZE))..."
|
||||
|
||||
clickhouse-client --password "$PASSWORD" --query="
|
||||
INSERT INTO default.$replicated_table
|
||||
SELECT * FROM remote('localhost', 'default', '$table')
|
||||
LIMIT $BATCH_SIZE OFFSET $offset
|
||||
SETTINGS
|
||||
max_insert_threads=$MAX_THREADS,
|
||||
max_memory_usage='10Gi'"
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error copying batch $((offset/BATCH_SIZE+1))"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# 3. Проверяем количество скопированных строк
|
||||
copied_rows=$(clickhouse-client --password "$PASSWORD" --query="
|
||||
SELECT count() FROM default.$replicated_table")
|
||||
|
||||
echo "Successfully copied $copied_rows/$total_rows rows from $table to $replicated_table"
|
||||
done
|
||||
```
|
||||
|
||||
#### 2. Копирование с помощью программы clickhouse-copier
|
||||
Создается конфиг подключения к кластеру (keeper)
|
||||
config.xml
|
||||
```xml
|
||||
<yandex>
|
||||
<logger>
|
||||
<level>warning</level>
|
||||
<log>/var/log/clickhouse-server/clickhouse-copier.log</log>
|
||||
<errorlog>/var/log/clickhouse-server/clickhouse-copier.err.log</errorlog>
|
||||
</logger>
|
||||
|
||||
<zookeeper>
|
||||
<!-- Для ClickHouse Keeper (рекомендуется) -->
|
||||
<node>
|
||||
<host>172.16.212.41</host>
|
||||
<port>9181</port>
|
||||
</node>
|
||||
</zookeeper>
|
||||
|
||||
<distributed_ddl>
|
||||
<path>/clickhouse/task_queue/ddl</path>
|
||||
</distributed_ddl>
|
||||
|
||||
<http_port>9999</http_port>
|
||||
</yandex>
|
||||
```
|
||||
|
||||
Создается папка tasks и там размещаются файлы для каждой таблицы, например api_request_log.xml
|
||||
```xml
|
||||
<yandex>
|
||||
<!-- Configuration of clusters as in an ordinary server config -->
|
||||
<remote_servers>
|
||||
<source_cluster>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>172.16.212.14</host>
|
||||
<port>9000</port>
|
||||
<user>default</user>
|
||||
<password>q1</password>
|
||||
<secure>0</secure>
|
||||
</replica>
|
||||
</shard>
|
||||
</source_cluster>
|
||||
|
||||
<destination_cluster>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>172.16.212.14</host>
|
||||
<port>9000</port>
|
||||
<user>default</user>
|
||||
<password>q1</password>
|
||||
<secure>0</secure>
|
||||
</replica>
|
||||
</shard>
|
||||
</destination_cluster>
|
||||
</remote_servers>
|
||||
|
||||
<!-- How many simultaneously active workers are possible. If you run more workers superfluous workers will sleep. -->
|
||||
<max_workers>2</max_workers>
|
||||
|
||||
<!-- Setting used to fetch (pull) data from source cluster tables -->
|
||||
<settings_pull>
|
||||
<readonly>1</readonly>
|
||||
</settings_pull>
|
||||
|
||||
<!-- Setting used to insert (push) data to destination cluster tables -->
|
||||
<settings_push>
|
||||
<readonly>0</readonly>
|
||||
</settings_push>
|
||||
|
||||
<!-- Common setting for fetch (pull) and insert (push) operations. Also, copier process context uses it.
|
||||
They are overlaid by <settings_pull/> and <settings_push/> respectively. -->
|
||||
<settings>
|
||||
<connect_timeout>3</connect_timeout>
|
||||
<!-- Sync insert is set forcibly, leave it here just in case. -->
|
||||
<insert_distributed_sync>1</insert_distributed_sync>
|
||||
</settings>
|
||||
|
||||
<tables>
|
||||
<!-- A table task, copies one table. -->
|
||||
<event>
|
||||
<!-- Source cluster name (from <remote_servers/> section) and tables in it that should be copied -->
|
||||
<cluster_pull>source_cluster</cluster_pull>
|
||||
<database_pull>default</database_pull>
|
||||
<table_pull>api_request_log</table_pull>
|
||||
|
||||
<!-- Destination cluster name and tables in which the data should be inserted -->
|
||||
<cluster_push>destination_cluster</cluster_push>
|
||||
<database_push>default</database_push>
|
||||
<table_push>api_request_log_replicated</table_push>
|
||||
|
||||
<engine>
|
||||
ENGINE = ReplicatedReplacingMergeTree(
|
||||
'/clickhouse/tables/{shard}/default/api_request_log_replicated',
|
||||
'{replica}',
|
||||
ver
|
||||
)
|
||||
PARTITION BY toYYYYMM(request_ts)
|
||||
ORDER BY uuid
|
||||
SETTINGS index_granularity = 8192;
|
||||
</engine>
|
||||
<sharding_key>rand()</sharding_key>
|
||||
</event>
|
||||
</tables>
|
||||
</yandex>
|
||||
```
|
||||
|
||||
В каждом файле меняется название базы в разделе **tables**
|
||||
После этого запускается копирование, где --task-path должен быть уникальным для каждой таблицы
|
||||
```bash
|
||||
clickhouse-copier --config config.xml --task-file ./tasks/api_request_log.xml --task-path /clickhouse/copier/task1
|
||||
```
|
||||
Loading…
Reference in New Issue