Class ColumnFetcher

class ColumnFetcher

Public Functions

ColumnFetcher(Executor *executor, const ColumnCacheMap &column_cache)

const int8_t *getOneTableColumnFragment(const int table_id, const int frag_id, const int col_id, const std::map<int, const TableFragments *> &all_tables_fragments, std::list<std::shared_ptr<Chunk_NS::Chunk>> &chunk_holder, std::list<ChunkIter> &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const

const int8_t *getAllTableColumnFragments(const int table_id, const int col_id, const std::map<int, const TableFragments *> &all_tables_fragments, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx) const

const int8_t *getResultSetColumn(const InputColDescriptor *col_desc, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx) const

const int8_t *linearizeColumnFragments(const int table_id, const int col_id, const std::map<int, const TableFragments *> &all_tables_fragments, std::list<std::shared_ptr<Chunk_NS::Chunk>> &chunk_holder, std::list<ChunkIter> &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx) const

void freeTemporaryCpuLinearizedIdxBuf()

void freeLinearizedBuf()

Public Static Functions

std::pair<const int8_t *, size_t> getOneColumnFragment(Executor *executor, const Analyzer::ColumnVar &hash_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx, std::vector<std::shared_ptr<Chunk_NS::Chunk>> &chunks_owner, ColumnCacheMap &column_cache)

Gets one chunk’s pointer and element count on either CPU or GPU.

Gets a column fragment chunk on CPU or on GPU depending on the effective memory level parameter. For temporary tables, the chunk will be copied to the GPU if needed. Returns a buffer pointer and an element count.

JoinColumn makeJoinColumn(Executor *executor, const Analyzer::ColumnVar &hash_col, const std::vector<Fragmenter_Namespace::FragmentInfo> &fragments, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx, std::vector<std::shared_ptr<Chunk_NS::Chunk>> &chunks_owner, std::vector<std::shared_ptr<void>> &malloc_owner, ColumnCacheMap &column_cache)

Creates a JoinColumn struct containing an array of JoinChunk structs.

makeJoinColumn() creates a JoinColumn struct containing a array of JoinChunk structs, col_chunks_buff, malloced in CPU memory. Although the col_chunks_buff array is in CPU memory here, each JoinChunk struct contains an int8_t* pointer from getOneColumnFragment(), col_buff, that can point to either CPU memory or GPU memory depending on the effective_mem_lvl parameter. See also the fetchJoinColumn() function where col_chunks_buff is copied into GPU memory if needed. The malloc_owner parameter will have the malloced array appended. The chunks_owner parameter will be appended with the chunks.

Private Types

using DeviceMergedChunkIterMap = std::unordered_map<int, int8_t *>

using DeviceMergedChunkMap = std::unordered_map<int, AbstractBuffer *>

Private Functions

MergedChunk linearizeVarLenArrayColFrags(const Catalog_Namespace::Catalog &cat, std::list<std::shared_ptr<Chunk_NS::Chunk>> &chunk_holder, std::list<ChunkIter> &chunk_iter_holder, std::list<std::shared_ptr<Chunk_NS::Chunk>> &local_chunk_holder, std::list<ChunkIter> &local_chunk_iter_holder, std::list<size_t> &local_chunk_num_tuples, MemoryLevel memory_level, const ColumnDescriptor *cd, const int device_id, const size_t total_data_buf_size, const size_t total_idx_buf_size, const size_t total_num_tuples, DeviceAllocator *device_allocator, const size_t thread_idx) const

MergedChunk linearizeFixedLenArrayColFrags(const Catalog_Namespace::Catalog &cat, std::list<std::shared_ptr<Chunk_NS::Chunk>> &chunk_holder, std::list<ChunkIter> &chunk_iter_holder, std::list<std::shared_ptr<Chunk_NS::Chunk>> &local_chunk_holder, std::list<ChunkIter> &local_chunk_iter_holder, std::list<size_t> &local_chunk_num_tuples, MemoryLevel memory_level, const ColumnDescriptor *cd, const int device_id, const size_t total_data_buf_size, const size_t total_idx_buf_size, const size_t total_num_tuples, DeviceAllocator *device_allocator, const size_t thread_idx) const

void addMergedChunkIter(const InputColDescriptor col_desc, const int device_id, int8_t *chunk_iter_ptr) const

const int8_t *getChunkiter(const InputColDescriptor col_desc, const int device_id = 0) const

ChunkIter prepareChunkIter(AbstractBuffer *merged_data_buf, AbstractBuffer *merged_index_buf, ChunkIter &chunk_iter, bool is_true_varlen_type, const size_t total_num_tuples) const

const int8_t *getResultSetColumn(const ResultSetPtr &buffer, const int table_id, const int col_id, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx) const

Private Members

Executor *executor_

std::mutex columnar_fetch_mutex_

std::mutex varlen_chunk_fetch_mutex_

std::mutex linearization_mutex_

std::mutex chunk_list_mutex_

std::mutex linearized_col_cache_mutex_

ColumnCacheMap columnarized_table_cache_

std::unordered_map<InputColDescriptor, std::unique_ptr<const ColumnarResults>> columnarized_scan_table_cache_

std::unordered_map<InputColDescriptor, DeviceMergedChunkIterMap> linearized_multi_frag_chunk_iter_cache_

std::unordered_map<int, AbstractBuffer *> linearlized_temporary_cpu_index_buf_cache_

std::unordered_map<InputColDescriptor, DeviceMergedChunkMap> linearized_data_buf_cache_

std::unordered_map<InputColDescriptor, DeviceMergedChunkMap> linearized_idx_buf_cache_

Private Static Functions

const int8_t *transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)

Friends

friend ColumnFetcher::QueryCompilationDescriptor

friend ColumnFetcher::TableFunctionExecutionContext