query_settings.go 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. package clickhouse
  2. import (
  3. "fmt"
  4. "net/url"
  5. "strconv"
  6. "github.com/ClickHouse/clickhouse-go/lib/binary"
  7. )
  8. type querySettingType int
  9. // all possible query setting's data type
  10. const (
  11. uintQS querySettingType = iota + 1
  12. intQS
  13. boolQS
  14. timeQS
  15. )
  16. // description of single query setting
  17. type querySettingInfo struct {
  18. name string
  19. qsType querySettingType
  20. }
  21. // all possible query settings
  22. var querySettingList = []querySettingInfo{
  23. {"min_compress_block_size", uintQS},
  24. {"max_compress_block_size", uintQS},
  25. {"max_block_size", uintQS},
  26. {"max_insert_block_size", uintQS},
  27. {"min_insert_block_size_rows", uintQS},
  28. {"min_insert_block_size_bytes", uintQS},
  29. {"max_read_buffer_size", uintQS},
  30. {"max_distributed_connections", uintQS},
  31. {"max_query_size", uintQS},
  32. {"interactive_delay", uintQS},
  33. {"poll_interval", uintQS},
  34. {"distributed_connections_pool_size", uintQS},
  35. {"connections_with_failover_max_tries", uintQS},
  36. {"background_pool_size", uintQS},
  37. {"background_schedule_pool_size", uintQS},
  38. {"replication_alter_partitions_sync", uintQS},
  39. {"replication_alter_columns_timeout", uintQS},
  40. {"min_count_to_compile", uintQS},
  41. {"min_count_to_compile_expression", uintQS},
  42. {"group_by_two_level_threshold", uintQS},
  43. {"group_by_two_level_threshold_bytes", uintQS},
  44. {"aggregation_memory_efficient_merge_threads", uintQS},
  45. {"max_parallel_replicas", uintQS},
  46. {"parallel_replicas_count", uintQS},
  47. {"parallel_replica_offset", uintQS},
  48. {"merge_tree_min_rows_for_concurrent_read", uintQS},
  49. {"merge_tree_min_bytes_for_concurrent_read", uintQS},
  50. {"merge_tree_min_rows_for_seek", uintQS},
  51. {"merge_tree_min_bytes_for_seek", uintQS},
  52. {"merge_tree_coarse_index_granularity", uintQS},
  53. {"merge_tree_max_rows_to_use_cache", uintQS},
  54. {"merge_tree_max_bytes_to_use_cache", uintQS},
  55. {"mysql_max_rows_to_insert", uintQS},
  56. {"optimize_min_equality_disjunction_chain_length", uintQS},
  57. {"min_bytes_to_use_direct_io", uintQS},
  58. {"mark_cache_min_lifetime", uintQS},
  59. {"priority", uintQS},
  60. {"log_queries_cut_to_length", uintQS},
  61. {"max_concurrent_queries_for_user", uintQS},
  62. {"insert_quorum", uintQS},
  63. {"select_sequential_consistency", uintQS},
  64. {"table_function_remote_max_addresses", uintQS},
  65. {"read_backoff_max_throughput", uintQS},
  66. {"read_backoff_min_events", uintQS},
  67. {"output_format_pretty_max_rows", uintQS},
  68. {"output_format_pretty_max_column_pad_width", uintQS},
  69. {"output_format_parquet_row_group_size", uintQS},
  70. {"http_headers_progress_interval_ms", uintQS},
  71. {"input_format_allow_errors_num", uintQS},
  72. {"preferred_block_size_bytes", uintQS},
  73. {"max_replica_delay_for_distributed_queries", uintQS},
  74. {"preferred_max_column_in_block_size_bytes", uintQS},
  75. {"insert_distributed_timeout", uintQS},
  76. {"odbc_max_field_size", uintQS},
  77. {"max_rows_to_read", uintQS},
  78. {"max_bytes_to_read", uintQS},
  79. {"max_rows_to_group_by", uintQS},
  80. {"max_bytes_before_external_group_by", uintQS},
  81. {"max_rows_to_sort", uintQS},
  82. {"max_bytes_to_sort", uintQS},
  83. {"max_bytes_before_external_sort", uintQS},
  84. {"max_bytes_before_remerge_sort", uintQS},
  85. {"max_result_rows", uintQS},
  86. {"max_result_bytes", uintQS},
  87. {"min_execution_speed", uintQS},
  88. {"max_execution_speed", uintQS},
  89. {"min_execution_speed_bytes", uintQS},
  90. {"max_execution_speed_bytes", uintQS},
  91. {"max_columns_to_read", uintQS},
  92. {"max_temporary_columns", uintQS},
  93. {"max_temporary_non_const_columns", uintQS},
  94. {"max_subquery_depth", uintQS},
  95. {"max_pipeline_depth", uintQS},
  96. {"max_ast_depth", uintQS},
  97. {"max_ast_elements", uintQS},
  98. {"max_expanded_ast_elements", uintQS},
  99. {"readonly", uintQS},
  100. {"max_rows_in_set", uintQS},
  101. {"max_bytes_in_set", uintQS},
  102. {"max_rows_in_join", uintQS},
  103. {"max_bytes_in_join", uintQS},
  104. {"max_rows_to_transfer", uintQS},
  105. {"max_bytes_to_transfer", uintQS},
  106. {"max_rows_in_distinct", uintQS},
  107. {"max_bytes_in_distinct", uintQS},
  108. {"max_memory_usage", uintQS},
  109. {"max_memory_usage_for_user", uintQS},
  110. {"max_memory_usage_for_all_queries", uintQS},
  111. {"max_network_bandwidth", uintQS},
  112. {"max_network_bytes", uintQS},
  113. {"max_network_bandwidth_for_user", uintQS},
  114. {"max_network_bandwidth_for_all_users", uintQS},
  115. {"low_cardinality_max_dictionary_size", uintQS},
  116. {"max_fetch_partition_retries_count", uintQS},
  117. {"http_max_multipart_form_data_size", uintQS},
  118. {"max_partitions_per_insert_block", uintQS},
  119. {"max_threads", uintQS},
  120. {"optimize_skip_unused_shards_nesting", uintQS},
  121. {"force_optimize_skip_unused_shards", uintQS},
  122. {"force_optimize_skip_unused_shards_nesting", uintQS},
  123. {"network_zstd_compression_level", intQS},
  124. {"http_zlib_compression_level", intQS},
  125. {"distributed_ddl_task_timeout", intQS},
  126. {"extremes", boolQS},
  127. {"use_uncompressed_cache", boolQS},
  128. {"replace_running_query", boolQS},
  129. {"distributed_directory_monitor_batch_inserts", boolQS},
  130. {"optimize_move_to_prewhere", boolQS},
  131. {"compile", boolQS},
  132. {"allow_suspicious_low_cardinality_types", boolQS},
  133. {"compile_expressions", boolQS},
  134. {"distributed_aggregation_memory_efficient", boolQS},
  135. {"skip_unavailable_shards", boolQS},
  136. {"distributed_group_by_no_merge", boolQS},
  137. {"optimize_skip_unused_shards", boolQS},
  138. {"merge_tree_uniform_read_distribution", boolQS},
  139. {"force_index_by_date", boolQS},
  140. {"force_primary_key", boolQS},
  141. {"log_queries", boolQS},
  142. {"insert_deduplicate", boolQS},
  143. {"enable_http_compression", boolQS},
  144. {"http_native_compression_disable_checksumming_on_decompress", boolQS},
  145. {"output_format_write_statistics", boolQS},
  146. {"add_http_cors_header", boolQS},
  147. {"input_format_skip_unknown_fields", boolQS},
  148. {"input_format_with_names_use_header", boolQS},
  149. {"input_format_import_nested_json", boolQS},
  150. {"input_format_defaults_for_omitted_fields", boolQS},
  151. {"input_format_values_interpret_expressions", boolQS},
  152. {"output_format_json_quote_64bit_integers", boolQS},
  153. {"output_format_json_quote_denormals", boolQS},
  154. {"output_format_json_escape_forward_slashes", boolQS},
  155. {"output_format_pretty_color", boolQS},
  156. {"use_client_time_zone", boolQS},
  157. {"send_progress_in_http_headers", boolQS},
  158. {"fsync_metadata", boolQS},
  159. {"join_use_nulls", boolQS},
  160. {"fallback_to_stale_replicas_for_distributed_queries", boolQS},
  161. {"insert_distributed_sync", boolQS},
  162. {"insert_allow_materialized_columns", boolQS},
  163. {"optimize_throw_if_noop", boolQS},
  164. {"use_index_for_in_with_subqueries", boolQS},
  165. {"empty_result_for_aggregation_by_empty_set", boolQS},
  166. {"allow_distributed_ddl", boolQS},
  167. {"join_any_take_last_row", boolQS},
  168. {"format_csv_allow_single_quotes", boolQS},
  169. {"format_csv_allow_double_quotes", boolQS},
  170. {"log_profile_events", boolQS},
  171. {"log_query_settings", boolQS},
  172. {"log_query_threads", boolQS},
  173. {"enable_optimize_predicate_expression", boolQS},
  174. {"low_cardinality_use_single_dictionary_for_part", boolQS},
  175. {"decimal_check_overflow", boolQS},
  176. {"prefer_localhost_replica", boolQS},
  177. //{"asterisk_left_columns_only", boolQS},
  178. {"calculate_text_stack_trace", boolQS},
  179. {"allow_ddl", boolQS},
  180. {"parallel_view_processing", boolQS},
  181. {"enable_debug_queries", boolQS},
  182. {"enable_unaligned_array_join", boolQS},
  183. {"low_cardinality_allow_in_native_format", boolQS},
  184. {"allow_experimental_multiple_joins_emulation", boolQS},
  185. {"allow_experimental_cross_to_join_conversion", boolQS},
  186. {"cancel_http_readonly_queries_on_client_close", boolQS},
  187. {"external_table_functions_use_nulls", boolQS},
  188. {"allow_experimental_data_skipping_indices", boolQS},
  189. {"allow_hyperscan", boolQS},
  190. {"allow_simdjson", boolQS},
  191. {"connect_timeout", timeQS},
  192. {"connect_timeout_with_failover_ms", timeQS},
  193. {"receive_timeout", timeQS},
  194. {"send_timeout", timeQS},
  195. {"tcp_keep_alive_timeout", timeQS},
  196. {"queue_max_wait_ms", timeQS},
  197. {"distributed_directory_monitor_sleep_time_ms", timeQS},
  198. {"insert_quorum_timeout", timeQS},
  199. {"read_backoff_min_latency_ms", timeQS},
  200. {"read_backoff_min_interval_between_events_ms", timeQS},
  201. {"stream_flush_interval_ms", timeQS},
  202. {"stream_poll_timeout_ms", timeQS},
  203. {"http_connection_timeout", timeQS},
  204. {"http_send_timeout", timeQS},
  205. {"http_receive_timeout", timeQS},
  206. {"max_execution_time", timeQS},
  207. {"timeout_before_checking_execution_speed", timeQS},
  208. }
  209. type querySettingValueEncoder func(enc *binary.Encoder) error
  210. type querySettings struct {
  211. settings map[string]querySettingValueEncoder
  212. settingsStr string // used for debug output
  213. }
  214. func makeQuerySettings(query url.Values) (*querySettings, error) {
  215. qs := &querySettings{
  216. settings: make(map[string]querySettingValueEncoder),
  217. settingsStr: "",
  218. }
  219. for _, info := range querySettingList {
  220. valueStr := query.Get(info.name)
  221. if valueStr == "" {
  222. continue
  223. }
  224. switch info.qsType {
  225. case uintQS, intQS, timeQS:
  226. value, err := strconv.ParseUint(valueStr, 10, 64)
  227. if err != nil {
  228. return nil, err
  229. }
  230. qs.settings[info.name] = func(enc *binary.Encoder) error { return enc.Uvarint(value) }
  231. case boolQS:
  232. valueBool, err := strconv.ParseBool(valueStr)
  233. if err != nil {
  234. return nil, err
  235. }
  236. value := uint64(0)
  237. if valueBool {
  238. value = 1
  239. }
  240. qs.settings[info.name] = func(enc *binary.Encoder) error { return enc.Uvarint(value) }
  241. default:
  242. err := fmt.Errorf("query setting %s has unsupported data type", info.name)
  243. return nil, err
  244. }
  245. if qs.settingsStr != "" {
  246. qs.settingsStr += "&"
  247. }
  248. qs.settingsStr += info.name + "=" + valueStr
  249. }
  250. return qs, nil
  251. }
  252. func (qs *querySettings) IsEmpty() bool {
  253. return len(qs.settings) == 0
  254. }
  255. func (qs *querySettings) Serialize(enc *binary.Encoder) error {
  256. for name, fn := range qs.settings {
  257. if err := enc.String(name); err != nil {
  258. return err
  259. }
  260. if err := fn(enc); err != nil {
  261. return err
  262. }
  263. }
  264. return nil
  265. }