Class foreign_storage::TextFileBufferParser

class TextFileBufferParser

Subclassed by foreign_storage::CsvFileBufferParser, foreign_storage::RegexFileBufferParser

Public Functions

virtual ParseBufferResult parseBuffer(ParseBufferRequest &request, bool convert_data_blocks, bool columns_are_pre_filtered = false) const = 0

Parses a given file buffer and returns data blocks for each column in the file along with metadata related to rows and row offsets within the buffer.

Parameters

  • convert_data_blocks: - convert import buffers to data blocks

  • columns_are_pre_filtered: - file buffer passed into parse_buffer only has the necessary columns that are being requested, not all columns.

virtual import_export::CopyParams validateAndGetCopyParams(const ForeignTable *foreign_table) const = 0

Validates foreign table parse options and returns a CopyParams object upon successful validation. An exception is thrown if validation fails.

virtual size_t findRowEndPosition(size_t &alloc_size, std::unique_ptr<char[]> &buffer, size_t &buffer_size, const import_export::CopyParams &copy_params, const size_t buffer_first_row_index, unsigned int &num_rows_in_buffer, FileReader *file_reader) const = 0

Finds and returns the offset of the end of the last row in the given buffer. If the buffer does not contain at least one row, the buffer is extended with more content from the file until a row is read. An exception is thrown if the buffer is extended to a maximum threshold and at least one row has still not been read.

virtual void validateFiles(const FileReader *file_reader, const ForeignTable *foreign_table) const = 0

Performs basic validation of files to be parsed.

Public Static Functions

std::map<int, DataBlockPtr> convertImportBuffersToDataBlocks(const std::vector<std::unique_ptr<import_export::TypedImportBuffer>> &import_buffers)
bool isCoordinateScalar(const std::string_view datum)
void processGeoColumn(std::vector<std::unique_ptr<import_export::TypedImportBuffer>> &import_buffers, size_t &col_idx, const import_export::CopyParams &copy_params, std::list<const ColumnDescriptor *>::iterator &cd_it, std::vector<std::string_view> &row, size_t &import_idx, bool is_null, size_t first_row_index, size_t row_index_plus_one, std::shared_ptr<Catalog_Namespace::Catalog> catalog, const RenderGroupAnalyzerMap *render_group_analyzer_map)
void fillRejectedRowWithInvalidData(const std::list<const ColumnDescriptor *> &columns, std::list<const ColumnDescriptor *>::iterator &cd_it, const size_t col_idx, ParseBufferRequest &request)

Fill the current row of the request with invalid (null) data as row will be marked as rejected

bool isNullDatum(const std::string_view datum, const ColumnDescriptor *column, const std::string &null_indicator)

Private Static Functions

void processInvalidGeoColumn(std::vector<std::unique_ptr<import_export::TypedImportBuffer>> &import_buffers, size_t &col_idx, const import_export::CopyParams &copy_params, const ColumnDescriptor *cd, std::shared_ptr<Catalog_Namespace::Catalog> catalog)