1 /* This is part of the netCDF package. 2 Copyright 2011 University Corporation for Atmospheric Research/Unidata 3 See COPYRIGHT file for conditions of use. 4 5 Test netcdf-4 chunking. 6 */ 7 8 #include <nc_tests.h> 9 #include "err_macros.h" 10 11 #define FILE_NAME "tst_chunks2.nc" 12 #define MAX_WASTE 25.0 13 #define NUM_RANDOM_TESTS 3 14 #define NDIMS3 3 15 16 /* Calculate the waste of the chunking. A waste of 10% means the 17 * chunked data is 10% larget then the unchunked data. */ 18 static int 19 calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) 20 { 21 int d; 22 float chunked = 1, unchunked = 1; 23 size_t *num_chunks; 24 size_t chunk_size = 1; 25 26 assert(waste && dimlen && chunksize && ndims); 27 if (!(num_chunks = calloc(ndims, sizeof(size_t)))) ERR; 28 29 #ifdef PRINT_CHUNK_WASTE_REPORT 30 printf("\n"); 31 #endif 32 /* Caclulate the total space taken up by the chunked data. */ 33 for (d = 0; d < ndims; d++) 34 { 35 /* How many chunks along this dimension are required to hold all the data? */ 36 for (num_chunks[d] = 0; (num_chunks[d] * chunksize[d]) < (dimlen[d] ? dimlen[d] : 1); 37 num_chunks[d]++) 38 ; 39 chunked *= (num_chunks[d] * chunksize[d]); 40 } 41 42 /* Calculate the minimum space required for this data 43 * (i.e. unchunked) or one record of it. */ 44 for (d = 0; d < ndims; d++) 45 unchunked *= (dimlen[d] ? dimlen[d] : 1); 46 47 #ifdef PRINT_CHUNK_WASTE_REPORT 48 printf("size for unchunked %g elements; size for chunked %g elements\n", 49 unchunked, chunked); 50 #endif 51 52 /* Percent of the chunked file that is wasted space. */ 53 *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0; 54 55 #ifdef PRINT_CHUNK_WASTE_REPORT 56 printf("\ndimlen\tchunksize\tnum_chunks\n"); 57 #endif 58 for (d = 0; d < ndims; d++) 59 { 60 #ifdef PRINT_CHUNK_WASTE_REPORT 61 printf("%ld\t%ld\t\t%ld\n", (long int)dimlen[d], (long int)chunksize[d], 62 (long int)num_chunks[d]); 63 #endif 64 chunk_size *= chunksize[d]; 65 } 66 #ifdef PRINT_CHUNK_WASTE_REPORT 67 printf("size of chunk: %ld elements; wasted space: %2.2f percent\n", 68 (long int)chunk_size, *waste); 69 #endif 70 71 free(num_chunks); 72 return 0; 73 } 74 75 int 76 main(int argc, char **argv) 77 { 78 printf("\n*** Testing netcdf-4 variable chunking.\n"); 79 printf("**** testing default chunksizes..."); 80 { 81 #define NDIMS3 3 82 #define NUM_VARS 1 83 #define Y_NAME "y" 84 #define X_NAME "x" 85 #define Z_NAME "z" 86 #define VAR_NAME_JOE "joe" 87 #define XDIM_LEN 2 88 #define YDIM_LEN 5 89 #define ZDIM_LEN 3000 90 91 int varid, ncid, dims[NDIMS3], dims_in[NDIMS3]; 92 int ndims, nvars, ngatts, unlimdimid, natts; 93 char name_in[NC_MAX_NAME + 1]; 94 nc_type type_in; 95 size_t len_in[NDIMS3]; 96 int storage = 0; 97 size_t chunksizes[NDIMS3]; 98 float waste = 0; 99 100 /* Create a file with 3D var, turn on chunking, but don't provide chunksizes. */ 101 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 102 if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; 103 if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; 104 if (nc_def_dim(ncid, Z_NAME, ZDIM_LEN, &dims[2])) ERR; 105 if (nc_def_var(ncid, VAR_NAME_JOE, NC_FLOAT, NDIMS3, dims, &varid)) ERR; 106 if (nc_def_var_chunking(ncid, 0, NC_CHUNKED, NULL)) ERR; 107 108 /* Check it out. */ 109 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; 110 if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR; 111 if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; 112 if (strcmp(name_in, VAR_NAME_JOE) || type_in != NC_FLOAT || ndims != NDIMS3 || 113 dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR; 114 if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR; 115 if (strcmp(name_in, X_NAME) || len_in[0] != XDIM_LEN) ERR; 116 if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR; 117 if (strcmp(name_in, Y_NAME) || len_in[1] != YDIM_LEN) ERR; 118 if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR; 119 if (strcmp(name_in, Z_NAME) || len_in[2] != ZDIM_LEN) ERR; 120 if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR; 121 if (storage != NC_CHUNKED) ERR; 122 if (nc_close(ncid)) ERR; 123 124 /* Open the file and check again. */ 125 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; 126 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; 127 if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR; 128 if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; 129 if (strcmp(name_in, VAR_NAME_JOE) || type_in != NC_FLOAT || ndims != NDIMS3 || 130 dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR; 131 if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR; 132 if (strcmp(name_in, X_NAME) || len_in[0] != XDIM_LEN) ERR; 133 if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR; 134 if (strcmp(name_in, Y_NAME) || len_in[1] != YDIM_LEN) ERR; 135 if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR; 136 if (strcmp(name_in, Z_NAME) || len_in[2] != ZDIM_LEN) ERR; 137 if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR; 138 if (storage != NC_CHUNKED) ERR; 139 if (calculate_waste(NDIMS3, len_in, chunksizes, &waste)) ERR; 140 /*if (waste > MAX_WASTE) ERR;*/ 141 if (nc_close(ncid)) ERR; 142 } 143 SUMMARIZE_ERR; 144 printf("**** testing default chunksizes some more for a 3D var..."); 145 { 146 #define NDIMS3 3 147 #define VAR_NAME "op-amp" 148 149 int varid, ncid; 150 int dimids[NDIMS3]; 151 size_t dim_len[NDIMS3] = {1, 11, 152750}; 152 153 int storage = 0; 154 size_t chunksizes[NDIMS3]; 155 int d; 156 char dim_name[NC_MAX_NAME + 1]; 157 float waste; 158 159 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 160 161 /* Create a few dimensions. */ 162 for (d = 0; d < NDIMS3; d++) 163 { 164 sprintf(dim_name, "dim_%d", d); 165 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 166 } 167 168 /* Define a var with these dimensions, and turn on chunking. */ 169 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 170 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 171 172 /* Check how default chunking worked. */ 173 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 174 if (storage != NC_CHUNKED) ERR; 175 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 176 /* if (waste > MAX_WASTE) ERR;*/ 177 178 if (nc_close(ncid)) ERR; 179 180 /* Open the file and check. */ 181 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; 182 if (nc_close(ncid)) ERR; 183 } 184 SUMMARIZE_ERR; 185 printf("**** testing default chunksizes even more for a 3D var..."); 186 { 187 int varid, ncid; 188 int dimids[NDIMS3]; 189 size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777}; 190 191 int storage = 0; 192 size_t chunksizes[NDIMS3]; 193 int d; 194 char dim_name[NC_MAX_NAME + 1]; 195 float waste; 196 197 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 198 199 /* Create a few dimensions. */ 200 for (d = 0; d < NDIMS3; d++) 201 { 202 sprintf(dim_name, "dim_%d", d); 203 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 204 } 205 206 /* Define a var with these dimensions, and turn on chunking. */ 207 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 208 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 209 210 /* Check how default chunking worked. */ 211 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 212 if (storage != NC_CHUNKED) ERR; 213 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 214 /* if (waste > MAX_WASTE) ERR;*/ 215 216 if (nc_close(ncid)) ERR; 217 218 /* Open the file and check. */ 219 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; 220 if (nc_close(ncid)) ERR; 221 } 222 SUMMARIZE_ERR; 223 printf("**** testing default chunksizes even even more for a 3D var..."); 224 { 225 int varid, ncid; 226 int dimids[NDIMS3]; 227 size_t dim_len[NDIMS3] = {1714636915, 1957747793, 424238335}; 228 229 int storage = 0; 230 size_t chunksizes[NDIMS3]; 231 int d; 232 char dim_name[NC_MAX_NAME + 1]; 233 float waste; 234 235 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 236 237 /* Create a few dimensions. */ 238 for (d = 0; d < NDIMS3; d++) 239 { 240 sprintf(dim_name, "dim_%d", d); 241 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 242 } 243 244 /* Define a var with these dimensions, and turn on chunking. */ 245 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 246 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 247 248 /* Check how default chunking worked. */ 249 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 250 if (storage != NC_CHUNKED) ERR; 251 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 252 /* if (waste > MAX_WASTE) ERR;*/ 253 254 if (nc_close(ncid)) ERR; 255 256 /* Open the file and check. */ 257 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; 258 if (nc_close(ncid)) ERR; 259 } 260 SUMMARIZE_ERR; 261 printf("**** testing default chunksizes some more for a 3D var..."); 262 { 263 #define NDIMS3 3 264 #define VAR_NAME "op-amp" 265 266 int varid, ncid; 267 int dimids[NDIMS3]; 268 size_t dim_len[NDIMS3] = {1967513926, 1365180540, 426}; 269 270 int storage = 0; 271 size_t chunksizes[NDIMS3]; 272 int d; 273 char dim_name[NC_MAX_NAME + 1]; 274 float waste; 275 276 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 277 278 /* Create a few dimensions. */ 279 for (d = 0; d < NDIMS3; d++) 280 { 281 sprintf(dim_name, "dim_%d", d); 282 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 283 } 284 285 /* Define a var with these dimensions, and turn on chunking. */ 286 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 287 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 288 289 /* Check how default chunking worked. */ 290 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 291 if (storage != NC_CHUNKED) ERR; 292 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 293 /* if (waste > MAX_WASTE) ERR;*/ 294 295 if (nc_close(ncid)) ERR; 296 297 /* Open the file and check. */ 298 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; 299 if (nc_close(ncid)) ERR; 300 } 301 SUMMARIZE_ERR; 302 printf("**** testing default chunksizes for very large 3D var..."); 303 { 304 #define NDIMS3 3 305 306 int varid, ncid; 307 int dimids[NDIMS3]; 308 size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777}; 309 310 int storage = 0; 311 size_t chunksizes[NDIMS3]; 312 int d; 313 char dim_name[NC_MAX_NAME + 1]; 314 float waste; 315 316 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 317 318 /* Create a few dimensions. */ 319 for (d = 0; d < NDIMS3; d++) 320 { 321 sprintf(dim_name, "dim_%d", d); 322 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 323 } 324 325 /* Define a var with these dimensions, and turn on chunking. */ 326 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 327 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 328 329 /* Check how default chunking worked. */ 330 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 331 if (storage != NC_CHUNKED) ERR; 332 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 333 /* if (waste > MAX_WASTE) ERR;*/ 334 335 if (nc_close(ncid)) ERR; 336 337 /* Open the file and check. */ 338 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; 339 if (nc_close(ncid)) ERR; 340 } 341 SUMMARIZE_ERR; 342 printf("**** testing default chunksizes some randomly sized 3D vars..."); 343 { 344 #define NDIMS3 3 345 346 int varid, ncid; 347 int dimids[NDIMS3]; 348 size_t dim_len[NDIMS3]; 349 int storage = 0; 350 size_t chunksizes[NDIMS3]; 351 int d, t; 352 char dim_name[NC_MAX_NAME + 1]; 353 float waste; 354 355 for (t = 0; t < NUM_RANDOM_TESTS; t++) 356 { 357 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 358 359 /* Create a few dimensions. */ 360 for (d = 0; d < NDIMS3; d++) 361 { 362 dim_len[d] = rand(); 363 sprintf(dim_name, "dim_%d", d); 364 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 365 } 366 367 /* Define a var with these dimensions, and turn on chunking. */ 368 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 369 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 370 371 /* Check how well default chunking worked. */ 372 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 373 if (storage != NC_CHUNKED) ERR; 374 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 375 if (waste > MAX_WASTE) ERR; 376 377 if (nc_close(ncid)) ERR; 378 } 379 } 380 SUMMARIZE_ERR; 381 printf("**** testing default chunksizes some randomly sized 3D vars, with one small dimension..."); 382 { 383 int varid, ncid; 384 int dimids[NDIMS3]; 385 size_t dim_len[NDIMS3]; 386 int storage = 0; 387 size_t chunksizes[NDIMS3]; 388 int d, t; 389 char dim_name[NC_MAX_NAME + 1]; 390 float waste; 391 392 for (t = 0; t < NUM_RANDOM_TESTS; t++) 393 { 394 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 395 396 dim_len[0] = rand(); 397 dim_len[1] = rand(); 398 dim_len[2] = rand() % 1000; 399 /* Create a few dimensions. */ 400 for (d = 0; d < NDIMS3; d++) 401 { 402 sprintf(dim_name, "dim_%d", d); 403 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 404 } 405 406 /* Define a var with these dimensions, and turn on chunking. */ 407 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 408 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 409 410 /* Check how well default chunking worked. */ 411 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 412 if (storage != NC_CHUNKED) ERR; 413 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 414 if (waste > MAX_WASTE) ERR; 415 416 if (nc_close(ncid)) ERR; 417 } 418 } 419 SUMMARIZE_ERR; 420 printf("**** testing default chunksizes some randomly sized 3D vars, with two small dimensions..."); 421 { 422 int varid, ncid; 423 int dimids[NDIMS3]; 424 size_t dim_len[NDIMS3]; 425 int storage = 0; 426 size_t chunksizes[NDIMS3]; 427 int d, t; 428 char dim_name[NC_MAX_NAME + 1]; 429 float waste; 430 431 for (t = 0; t < NUM_RANDOM_TESTS; t++) 432 { 433 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; 434 435 dim_len[0] = rand(); 436 dim_len[1] = rand() % 1000; 437 dim_len[2] = rand() % 1000; 438 /* Create a few dimensions. */ 439 for (d = 0; d < NDIMS3; d++) 440 { 441 sprintf(dim_name, "dim_%d", d); 442 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; 443 } 444 445 /* Define a var with these dimensions, and turn on chunking. */ 446 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; 447 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; 448 449 /* Check how well default chunking worked. */ 450 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; 451 if (storage != NC_CHUNKED) ERR; 452 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; 453 if (waste > MAX_WASTE) ERR; 454 455 if (nc_close(ncid)) ERR; 456 } 457 } 458 SUMMARIZE_ERR; 459 FINAL_RESULTS; 460 }
HyperKWIC - Version 7.20DA executed at 11:37 on 27 Oct 2017 | Polyhedron Solutions - INTERNAL USE | COMMERCIAL (Any O/S) SN 4AKIed