From 5cb2239d7db6d41811565e425353af2d599ab1d3 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Thu, 4 Jun 2015 12:58:30 +0100 Subject: [PATCH] try to support DOS ppm/csv files on linux swap \r\n for \n in vips__fgetc() --- ChangeLog | 1 + TODO | 15 ++------------- libvips/foreign/csv.c | 23 +++++++++++++---------- libvips/foreign/ppm.c | 13 ++++++++----- libvips/include/vips/util.h | 1 + libvips/iofuncs/util.c | 23 +++++++++++++++++++++++ 6 files changed, 48 insertions(+), 28 deletions(-) diff --git a/ChangeLog b/ChangeLog index 639c4e8e..33ed2fdd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ - oops, base64 encode could pad by up to two zero bytes - added VipsRefString as a thing that gi bindings can unpack - support "with Vips.Image as ...:" in Python +- try to support DOS CSV and PPM files on linux 7/5/15 started 8.0.3 - dzsave and tif pyr write could fail for some image dimensions, thanks Jonas diff --git a/TODO b/TODO index 52209931..2368ed26 100644 --- a/TODO +++ b/TODO @@ -1,21 +1,10 @@ -- try: - - http://i.publiclab.org/system/images/photos/000/006/146/original/NDVI_VGYRM-lut.txt - - tab-separated, four columns - - $ vips csvload ~/Desktop/NDVI_VGYRM-lut.txt x.v --skip 1 - vips warning: csv2vips: error parsing number, line 2, column 1025 - - strange ... works without the --skip, though the first row is blank - - try: $ vips copy y.v x.v --width 1 --bands 3 VipsRegion: images do not match in pixel size - copy will make 1xN or Nx1 M-band images into MxN oone-nband images, but it - won't go tyhe other way + copy will make 1xN or Nx1 M-band images into MxN one-band images, but it + won't go the other way diff --git a/libvips/foreign/csv.c b/libvips/foreign/csv.c index 913fee99..04c0f588 100644 --- a/libvips/foreign/csv.c +++ b/libvips/foreign/csv.c @@ -27,6 +27,9 @@ * - report positions for EOF/EOL errors * 2/7/13 * - add matrix read/write + * 4/6/15 + * - try to support DOS files under linux ... we have to look for \r\n + * linebreaks */ /* @@ -84,13 +87,13 @@ skip_line( FILE *fp ) /* Are we at a delayed EOF? See below. */ - if( (ch = fgetc( fp )) == EOF ) + if( (ch = vips__fgetc( fp )) == EOF ) return( 0 ); ungetc( ch, fp ); /* If we hit EOF and no \n, wait until the next call to report EOF. */ - while( (ch = fgetc( fp )) != '\n' && + while( (ch = vips__fgetc( fp )) != '\n' && ch != EOF ) ; @@ -103,7 +106,7 @@ skip_white( FILE *fp, const char whitemap[256] ) int ch; do { - ch = fgetc( fp ); + ch = vips__fgetc( fp ); } while( ch != EOF && ch != '\n' && whitemap[ch] ); @@ -119,12 +122,12 @@ skip_to_quote( FILE *fp ) int ch; do { - ch = fgetc( fp ); + ch = vips__fgetc( fp ); /* Ignore \" in strings. */ if( ch == '\\' ) - ch = fgetc( fp ); + ch = vips__fgetc( fp ); else if( ch == '"' ) break; } while( ch != EOF && @@ -141,7 +144,7 @@ skip_to_sep( FILE *fp, const char sepmap[256] ) int ch; do { - ch = fgetc( fp ); + ch = vips__fgetc( fp ); } while( ch != EOF && ch != '\n' && !sepmap[ch] ); @@ -181,9 +184,9 @@ read_double( FILE *fp, const char whitemap[256], const char sepmap[256], return( ch ); if( ch == '"' ) { - (void) fgetc( fp ); + (void) vips__fgetc( fp ); (void) skip_to_quote( fp ); - (void) fgetc( fp ); + (void) vips__fgetc( fp ); } else if( !sepmap[ch] && fscanf( fp, "%lf", out ) != 1 ) { @@ -207,7 +210,7 @@ read_double( FILE *fp, const char whitemap[256], const char sepmap[256], */ if( ch != EOF && sepmap[ch] ) - (void) fgetc( fp ); + (void) vips__fgetc( fp ); return( 0 ); } @@ -462,7 +465,7 @@ fetch_nonwhite( FILE *fp, const char whitemap[256], char *buf, int max ) int i; for( i = 0; i < max - 1; i++ ) { - ch = fgetc( fp ); + ch = vips__fgetc( fp ); if( ch == EOF || ch == '\n' || whitemap[ch] ) break; diff --git a/libvips/foreign/ppm.c b/libvips/foreign/ppm.c index 3d5fd772..61935bf2 100644 --- a/libvips/foreign/ppm.c +++ b/libvips/foreign/ppm.c @@ -31,6 +31,9 @@ * - rework as a set of fns ready to be called from a class * 8/11/14 * - add 1 bit write + * 4/6/15 + * - try to support DOS files under linux ... we have to look for \r\n + * linebreaks */ /* @@ -84,7 +87,7 @@ static void skip_line( FILE *fp ) { - while( fgetc( fp ) != '\n' ) + while( vips__fgetc( fp ) != '\n' ) ; } @@ -93,7 +96,7 @@ skip_white_space( FILE *fp ) { int ch; - while( isspace( ch = fgetc( fp ) ) ) + while( isspace( ch = vips__fgetc( fp ) ) ) ; ungetc( ch, fp ); @@ -163,8 +166,8 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) /* Read in the magic number. */ - buf[0] = fgetc( fp ); - buf[1] = fgetc( fp ); + buf[0] = vips__fgetc( fp ); + buf[1] = vips__fgetc( fp ); buf[2] = '\0'; for( index = 0; index < VIPS_NUMBER( magic_names ); index++ ) @@ -220,7 +223,7 @@ read_header( FILE *fp, VipsImage *out, int *bits, int *ascii, int *msb_first ) * character before the data starts. */ if( !*ascii && - !isspace( fgetc( fp ) ) ) { + !isspace( vips__fgetc( fp ) ) ) { vips_error( "ppm2vips", "%s", _( "not whitespace before start of binary data" ) ); return( -1 ); diff --git a/libvips/include/vips/util.h b/libvips/include/vips/util.h index 38c0499d..f7aaa800 100644 --- a/libvips/include/vips/util.h +++ b/libvips/include/vips/util.h @@ -207,6 +207,7 @@ char *vips__file_read_name( const char *name, const char *fallback_dir, size_t *length_out ); int vips__file_write( void *data, size_t size, size_t nmemb, FILE *stream ); int vips__get_bytes( const char *filename, unsigned char buf[], int len ); +int vips__fgetc( FILE *fp ); GValue *vips__gvalue_ref_string_new( const char *text ); void vips__gslist_gvalue_free( GSList *list ); diff --git a/libvips/iofuncs/util.c b/libvips/iofuncs/util.c index a43c4a31..7afafcb3 100644 --- a/libvips/iofuncs/util.c +++ b/libvips/iofuncs/util.c @@ -763,6 +763,29 @@ vips__get_bytes( const char *filename, unsigned char buf[], int len ) return( 1 ); } +/* We try to support stupid DOS files too. These have \r\n (13, 10) as line + * separators. Strategy: an fgetc() that swaps \r\n for \n. + * + * On Windows, stdio will automatically swap \r\n for \n, but on Linux we have + * to do this by hand. + */ +int +vips__fgetc( FILE *fp ) +{ + int ch; + + ch = fgetc( fp ); + if( ch == '\r' ) { + ch = fgetc( fp ); + if( ch != '\n' ) { + ungetc( ch, fp ); + ch = '\r'; + } + } + + return( ch ); +} + /* Alloc/free a GValue. */ static GValue *