From e911701ee0420728ac605e4b9008fb4f5c9bd196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henning=20P=C3=B6ttker?= <25299532+hpoettker@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:58:18 +0200 Subject: [PATCH] Support moved rows in SAS data files --- Makefile.am | 5 +- resources/compression_type_2.sas7bdat | Bin 0 -> 20480 bytes resources/compression_type_6.sas7bdat | Bin 0 -> 20480 bytes src/sas/readstat_sas.h | 4 + src/sas/readstat_sas7bdat_read.c | 122 +++++++++++++++++++++++++- src/test/test_list.h | 74 ++++++++++++++++ src/test/test_read.c | 18 ++-- src/test/test_types.h | 2 + 8 files changed, 213 insertions(+), 12 deletions(-) create mode 100644 resources/compression_type_2.sas7bdat create mode 100644 resources/compression_type_6.sas7bdat diff --git a/Makefile.am b/Makefile.am index faf22521..18b12c63 100644 --- a/Makefile.am +++ b/Makefile.am @@ -82,7 +82,10 @@ endif dist_man1_MANS = man/readstat.man man/extract_metadata.man -EXTRA_DIST = resources/datetime.sas7bdat +EXTRA_DIST = \ + resources/datetime.sas7bdat \ + resources/compression_type_2.sas7bdat \ + resources/compression_type_6.sas7bdat include_HEADERS = src/readstat.h diff --git a/resources/compression_type_2.sas7bdat b/resources/compression_type_2.sas7bdat new file mode 100644 index 0000000000000000000000000000000000000000..fc1ab2d6ae4ae6cd884274ed282dc57079ac3acf GIT binary patch literal 20480 zcmeHPJB(dN89w_kS%an6V2MZsF&IlwWUP5VXt0%-hh)HtT_iNlJc%X4TER&iDTshX z0iuX#cuCOGA{vT-5Dg_16i`tjMHCSQZ@heS@0q*1*WNe+p+snobnlt@=Rg1d&-Xv( z%spp!n)Z)hy!z#*eti1WuYUT?o$2^5`8OVV-^o+*%p~Q;Cf``GZEYN)%Z(4n>AllL zwpJ`xE?glW{luk<E?m9PM6_Z4t@r=ogI9ik;ZHw0vp6p7q`t6Bj|3h0dectW=e5OCt*=PFGZ(kCg*gVYt+VN$7wHKfHUzzEvE%!{nInyt# z$#*l|tRv7$3Fe|lh^dSUnED6PVRH5#eRUz88f@x12A&pi6LSbtp8p1)kVdhv35o>Q;iDCGI;x1J`S+kLKN^33k@U$_n- zDcmZb-F@bTo8g5UyCGk@wR`Q^n;FCd@)yl{;$82tdHeyOb+6+vYIk$I_~6~NgOn2s zyr!w+-L>@F+Vy_NrPJ5%=X$?m-ue1G*86v6#!jEjSMdOdNBf1~^c+a#hx6^w*zNZt~ z=IFrD{nF#4pAWE%=IbDE)PD4JF1hDW_e1w%@6X%PRUW=$zPjgk=j-R473V$G{1?QX z#k1;1rnYl_cu}uUEWS@KzR%7T{C*0yVc*7GRet}f^|;1JpW1@yJ4fvQS4=NA4L`e2 zwcXb@S_6TBz(8OiFc26B3_dEZ1Ufls2=YhaLU?4CM7zhjm1_A?tfxtjuATSU(gaDZ)a(0>+ z&K#eYwzl!!)HGc#&+Q)K8=W_dKz~=L@A8)ywl~&~;aUD+>JN_C|G&~-P{QuNy+d(& zele}TFSEI}|L^=`|Nn;FG#Cd01A&3SKwu#77DQlY{~dn0|7JM;nNA4V8fqP{R4B!$ zx7t&tlT_BIs7xX`D5bQiv`7a>+y>QHD~2wv@5z zsP{130|@G+N=d?QB@z)?yHFu1+Nx;^uqmogspZT&?X=Y~kg9DB3gd;#jEN}?>r|$I zDRFO$b|Mp*t@Nb|Xo`#q3&dTGkmN-0!We5^BHAexONNjJQK`y<9kY{J$;iDFLPcU@ zNmZsmv&+Dl0|HkAhfR@dn}e`X5UEor1|EYUf;tDoAt+H97|U8J19y^G96YD-|C78E zR1~NSz6i>l@`8uxaUv;ZR*Ybj*Tmub=9cE5$ZbIgB<6ias5l+C0D4Xa%1Z;2GlZ8= z$?;N)OP;9eq$|nO%(JGJsYty_k!IxzV}bi<3^9tcBqY%B&O>Jll~|!YC4^c{I$K0$ z0RTEVYO*WP*$|f%i1TE;w9#hni48$Z2MCO4I-M%iF)15j*0Q2RmJx4b?1iDqB1uuqBmTXaS=$i<$(Byb%_N%LK%=N;DR@B_mKE z$Xm-fk=!G1kS$0ZxE1+Dqt1#djv!+yz;1(hs?vC^;C7Q~ZV({aEkzFa2qzgX1wqs% z*dVAe0@kNyIAV;pz-+>W6P!4!i-i{IV-3_;*c3bvk2z-Ud^W*bBH@-|LZOH*gKSfR z#=sp_4j98|YgHu@u-cfAO*R1?hwz2r$cvPN&Y3ZnB9XD$=omtYOHEa7>Z(gLDIOG) zSsI>5u}K1bWQ#U1w_tWYp$H*>kQv+t6*;lRRVOS*N@9uy_)(NAjeqo&EiaKF?J|kF zVALYsB+M1Hr8!CwwFwy=4boC;qH<9#1%x7Jg=UR-h>J?63^FkV=b|DRr6qBL3>RJ^ zJe%P;BUGwF-H0ZJD2diUz~zDffKFSKgt{Opa!<)*6XHD*0ez_!MY+b1P7NpDDwR;& z#elTri%<~;fkSIaj4IQriU{zs2Q`%TC-q zAz=loZ(!W0fOs;fZ$YCLpk4{8tU>?@omr>^l}8~FD+-cPG^3W#z!8I16~r|FfpdWl{lM)m>dZ1Hk(B^FMs^sn@4>{PwNywm|z=`C}B+nUP_0w(9TmE(1PC^c+S)usM^%b2lZ8f-h-#QSxD6ridZq8J%`5%V9jn3d85GVu+fkNOvN?@fq38n|fmoA)qsEYd)oOxug;x3A zU0icTvh=~`p)FFZ-3*)jYbaNyuDxL zTf1+m5{FLO`6i>fEdbLay#Z>!Z*Lh2_07~M5mc^jz}0B}K2cems{EV-JJ!-x1OKQp zg?0Vjyk7I<7mlB&ynQZy z`N}gdU$rkkeZ_{0&t19r{MC@fb^oiOs^WFIo2v9Jnbz<d956;+5?m)^1j|j(Fwu zn5w>YDfMmbc)aFP>*M!xb-ZR?+tu|LkAE;$Zgn_owXNe-H@I>0vEMHIUgfXmQkOR# z|665uJ0Jg#+Mn|_tk*b)zvOMG*9(`c{b{JIZr!T~Mu9(74NR1PXydpb+>kA+V{Q-ERj!*M$wlC||HK4LD1PXydpb#ho3V}kP5GVu+fkL1VxP<@&0L}t{ z^Pc3-%49RYHwD1C_`;Q2>`LSl6R6)6s%QDD8HaDIZ^~==x2msgF#rE!{en`S{SV)v zI9|P&_Ex2D9-IHy_GSM6iJPey76OGpAy5bu0{_bdhQA7Et-tWkwm=sE2*Pp9k=HaJ zXE*@>8OjnOOhyx|pfSHZj-d{Yc*_{V&Zuld4G%~Y>5z@V*a=`UabCD%4UvSvNckoZ z1f@h9j`4^DNyGpUu^`-X$f;u{NDDg1IZ6?woFWM{GAZ9c9{h(cd%KLZ8$Qx8c8eE{c0;duR)+b~! zZD_)GF=D41+Iqe#_O_X*ur$f`TlyH#{U*1_p5CR5i^3?He?Or-fr=aJwb4jhYk^$c z+iP1w8QU+6v5ayC8qUEGlnzlFWHi3h9tR{tO@YWMIV-&25_7n>kz8DuS$IHVjZx5? zi=B8d@brEoq}PrE38_;FLqe0>CWJ!ValWU!8wsk~U{}I{I?kZ*gx;I@J^GR6Il$Br z-Ma&MMx5Y+YUd{25inApME{_iQVpzjA+;SN01_wIX%Z!HbK+;hgP0NA3;kRYXVH(abml>la7}_Ja)8y zRQC_@Xf-FfBU0IH1c(h^kV2rOiAT7lR+?rHVTj@21EV4V4(+ycTnQrf%N;`OjOvJC z+Ysiqb0Ek_XL5Ii8qXgwIdaHe0-nuCY}XX*LHrj{U>kyLlu5vB*J3B1c!fVEn&qqsunF=QlfzgeEEwO%%%0qi7b$WAW041T5io_5Z^T4w?tus9AL&0tv6w$? z@jM^w><;aDJ+2QAS!m4_pff$Z~|h|J{-w&+#JCu45O ztqay^E_-K`!_YLoM}=2W^>dqbBvDPiOQ>fFxK4%4(LhPNOB1ygg*3s0;M)^goFSW} oQO4`27cgi;oJVz~xJ1&cCe};K17PPNRnlOb`=1vU<}oz=6WbE;xBvhE literal 0 HcmV?d00001 diff --git a/src/sas/readstat_sas.h b/src/sas/readstat_sas.h index 5d589107..6f8a6a84 100644 --- a/src/sas/readstat_sas.h +++ b/src/sas/readstat_sas.h @@ -123,8 +123,12 @@ typedef enum sas_subheader_type_e { #define SAS_COMPRESSION_NONE 0x00 #define SAS_COMPRESSION_TRUNC 0x01 +#define SAS_COMPRESSION_NONE_MOVED 0x02 +#define SAS_COMPRESSION_REFERENCE 0x03 #define SAS_COMPRESSION_ROW 0x04 #define SAS_COMPRESSION_DELETED_ROW 0x05 +#define SAS_COMPRESSION_ROW_MOVED 0x06 +#define SAS_COMPRESSION_MYSTERY 0x0d #define SAS_COMPRESSION_SIGNATURE_RLE "SASYZCRL" #define SAS_COMPRESSION_SIGNATURE_RDC "SASYZCR2" diff --git a/src/sas/readstat_sas7bdat_read.c b/src/sas/readstat_sas7bdat_read.c index cca57e85..6845654b 100644 --- a/src/sas/readstat_sas7bdat_read.c +++ b/src/sas/readstat_sas7bdat_read.c @@ -43,6 +43,7 @@ typedef struct sas7bdat_ctx_s { readstat_io_t *io; int bswap; int did_submit_columns; + int requires_page_seek; uint32_t row_length; uint32_t page_row_count; @@ -965,7 +966,7 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC && shp_info.compression != SAS_COMPRESSION_REFERENCE) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -977,7 +978,9 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_ goto cleanup; } } - } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW) { + } else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW || + shp_info.compression == SAS_COMPRESSION_ROW_MOVED || shp_info.compression == SAS_COMPRESSION_MYSTERY || + shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; @@ -1012,6 +1015,97 @@ static readstat_error_t sas7bdat_parse_deleted_row_bitmap(const char *page, cons return READSTAT_OK; } +static readstat_error_t sas7bdat_parse_moved_row(uint64_t page_index, uint64_t subheader_index, sas7bdat_ctx_t *ctx) { + readstat_error_t retval = READSTAT_OK; + readstat_io_t *io = ctx->io; + + const uint64_t page_size = ctx->page_size; + char *page = NULL; + + if (page_index >= ctx->page_count) { + retval = READSTAT_ERROR_PARSE; + goto cleanup; + } + + ctx->requires_page_seek = 1; + if (io->seek(ctx->header_size + page_index * page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64 + " (= %" PRId64 " + %" PRId64 "*%" PRId64 ")", + ctx->header_size + page_index * page_size, ctx->header_size, page_index, page_size); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + goto cleanup; + } + if ((page = readstat_malloc(page_size)) == NULL) { + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + if (io->read(page, page_size, io->io_ctx) < page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + + uint16_t page_type = sas_read2(&page[ctx->page_header_size - 8], ctx->bswap); + if ((page_type & SAS_PAGE_TYPE_MASK) == SAS_PAGE_TYPE_DATA || page_type & SAS_PAGE_TYPE_COMP) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint16_t subheader_count = sas_read2(&page[ctx->page_header_size - 4], ctx->bswap); + if (subheader_index >= subheader_count) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + uint64_t shp_offset = ctx->page_header_size + subheader_index * ctx->subheader_pointer_size; + if (shp_offset + ctx->subheader_pointer_size >= page_size) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + + const char *shp = &page[shp_offset]; + subheader_pointer_t shp_info = { 0 }; + if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { + goto cleanup; + } + if ((retval = sas7bdat_submit_columns_if_needed(ctx, 1)) != READSTAT_OK) { + goto cleanup; + } + + if (shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { + sas_subheader_type_t subheader_type = sas7bdat_parse_subheader_type(page + shp_info.offset, ctx); + if (!shp_info.is_compressed_data || subheader_type != SAS_SUBHEADER_TYPE_DATA) { + retval = READSTAT_ERROR_READ; + goto cleanup; + } + if (shp_info.len != ctx->row_length) { + retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH; + goto cleanup; + } + if ((retval = sas7bdat_parse_single_row(page + shp_info.offset, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED) { + if ((retval = sas7bdat_parse_subheader_compressed(page + shp_info.offset, shp_info.len, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else { + retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; + goto cleanup; + } + +cleanup: + + if (page) { + free(page); + } + + return retval; +} + static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) { uint16_t page_type; @@ -1041,7 +1135,13 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_parse_subheader_pointer(shp, page + page_size - shp, &shp_info, ctx)) != READSTAT_OK) { goto cleanup; } - if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { + if (shp_info.len > 0 && shp_info.compression == SAS_COMPRESSION_REFERENCE) { + uint64_t page_index = shp_info.offset - 1; + uint64_t subheader_index = shp_info.len - 1; + if ((retval = sas7bdat_parse_moved_row(page_index, subheader_index, ctx)) != READSTAT_OK) { + goto cleanup; + } + } else if (shp_info.len > 0 && shp_info.compression != SAS_COMPRESSION_TRUNC) { if ((retval = sas7bdat_validate_subheader_pointer(&shp_info, page_size, subheader_count, ctx)) != READSTAT_OK) { goto cleanup; } @@ -1076,6 +1176,9 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_ if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) { goto cleanup; } + } else if (shp_info.compression == SAS_COMPRESSION_ROW_MOVED || shp_info.compression == SAS_COMPRESSION_MYSTERY + || shp_info.compression == SAS_COMPRESSION_NONE_MOVED) { + /* void */ } else { retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION; goto cleanup; @@ -1256,6 +1359,19 @@ static readstat_error_t sas7bdat_parse_all_pages_pass2(sas7bdat_ctx_t *ctx) { if ((retval = sas7bdat_update_progress(ctx)) != READSTAT_OK) { goto cleanup; } + if (ctx->requires_page_seek) { + if (io->seek(ctx->header_size + i * ctx->page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) { + retval = READSTAT_ERROR_SEEK; + if (ctx->handle.error) { + snprintf(ctx->error_buf, sizeof(ctx->error_buf), "ReadStat: Failed to seek to position %" PRId64 + " (= %" PRId64 " + %" PRId64 "*%" PRId64 ")", + ctx->header_size + i * ctx->page_size, ctx->header_size, i, ctx->page_size); + ctx->handle.error(ctx->error_buf, ctx->user_ctx); + } + goto cleanup; + } + ctx->requires_page_seek = 0; + } if (io->read(ctx->page, ctx->page_size, io->io_ctx) < ctx->page_size) { retval = READSTAT_ERROR_READ; goto cleanup; diff --git a/src/test/test_list.h b/src/test/test_list.h index 9b78c98d..0efe376c 100644 --- a/src/test/test_list.h +++ b/src/test/test_list.h @@ -2383,6 +2383,80 @@ static rt_test_group_t _test_groups[] = { } } } + }, + + { + .resource_name = "compression_type_2.sas7bdat", + .label = "SAS file with moved rows", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } + }, + + { + .resource_name = "compression_type_6.sas7bdat", + .label = "SAS file with moved rows", + .test_formats = RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS, + .rows = 5, + .columns_count = 3, + .columns = { + { + .name = "id", + .type = READSTAT_TYPE_DOUBLE, + .values = { + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } }, + { .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } } + } + }, + { + .name = "category", + .type = READSTAT_TYPE_STRING, + .values = { + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_1" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_2" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_3" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_4" } }, + { .type = READSTAT_TYPE_STRING, .v = { .string_value = "CATEGORY_5" } } + } + }, + { + .name = "payload_to_update", + .type = READSTAT_TYPE_STRING, + .skip_value_comparison = 1 + } + } } } } diff --git a/src/test/test_read.c b/src/test/test_read.c index 1bd93895..f46da39b 100644 --- a/src/test/test_read.c +++ b/src/test/test_read.c @@ -265,14 +265,16 @@ static int handle_value(int obs_index, readstat_variable_t *variable, readstat_v rt_column_t *column = &rt_ctx->file->columns[rt_ctx->var_index]; - if (column->type == READSTAT_TYPE_STRING_REF) { - push_error_if_strings_differ(rt_ctx, - rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], - readstat_string_value(value), "String ref values"); - } else { - push_error_if_values_differ(rt_ctx, - column->values[file_obs_index], - value, "Data values"); + if (!column->skip_value_comparison) { + if (column->type == READSTAT_TYPE_STRING_REF) { + push_error_if_strings_differ(rt_ctx, + rt_ctx->file->string_refs[readstat_int32_value(column->values[file_obs_index])], + readstat_string_value(value), "String ref values"); + } else { + push_error_if_values_differ(rt_ctx, + column->values[file_obs_index], + value, "Data values"); + } } return READSTAT_HANDLER_OK; diff --git a/src/test/test_types.h b/src/test/test_types.h index 57839333..d0862844 100644 --- a/src/test/test_types.h +++ b/src/test/test_types.h @@ -42,6 +42,8 @@ typedef struct rt_column_s { long missing_ranges_count; char label_set[RT_MAX_STRING]; + + int skip_value_comparison; } rt_column_t; typedef struct rt_test_file_s {