diff --git a/ChangeLog b/ChangeLog index 7359d1d4..1e4a0583 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,7 +5,7 @@ - allow \ as an escape character in vips_break_token() [akemrir] - tiffsave has a "depth" param to set max pyr depth - libtiff LOGLUV images load and save as libvips XYZ -- add gifload_source +- add gifload_source, csvload_source, csvsave_target - revise vipsthumbnail flags - add VIPS_LEAK env var - add vips_pipe_read_limit_set(), --vips-pipe-read-limit, diff --git a/libvips/deprecated/im_csv2vips.c b/libvips/deprecated/im_csv2vips.c index e4e5a860..38d0f928 100644 --- a/libvips/deprecated/im_csv2vips.c +++ b/libvips/deprecated/im_csv2vips.c @@ -61,6 +61,8 @@ im_csv2vips( const char *filename, IMAGE *out ) char mode[FILENAME_MAX]; char *p, *q, *r; + VipsImage *x; + /* Parse mode string. */ im_filename_split( filename, name, mode ); @@ -76,9 +78,18 @@ im_csv2vips( const char *filename, IMAGE *out ) lines = atoi( r ); } - if( vips__csv_read( name, out, - start_skip, lines, whitespace, separator, FALSE ) ) + if( vips_csvload( name, &x, + "skip", start_skip, + "lines", lines, + "whitespace", whitespace, + "separator", separator, + NULL ) ) return( -1 ); + if( vips_image_write( x, out ) ) { + g_object_unref( x ); + return( -1 ); + } + g_object_unref( x ); return( 0 ); } diff --git a/libvips/foreign/csv.c b/libvips/foreign/csv.c index 70aae08c..3875fb46 100644 --- a/libvips/foreign/csv.c +++ b/libvips/foreign/csv.c @@ -118,352 +118,6 @@ skip_white( FILE *fp, const char whitemap[256] ) return( ch ); } -static int -skip_to_quote( FILE *fp ) -{ - int ch; - - do { - ch = vips__fgetc( fp ); - - /* Ignore \" in strings. - */ - if( ch == '\\' ) - ch = vips__fgetc( fp ); - else if( ch == '"' ) - break; - } while( ch != EOF && - ch != '\n' ); - - ungetc( ch, fp ); - - return( ch ); -} - -static int -skip_to_sep( FILE *fp, const char sepmap[256] ) -{ - int ch; - - do { - ch = vips__fgetc( fp ); - } while( ch != EOF && - ch != '\n' && - !sepmap[ch] ); - - ungetc( ch, fp ); - - return( ch ); -} - -/* Read a single item. Syntax is: - * - * element : - * whitespace* item whitespace* [EOF|EOL|separator] - * - * item : - * double | - * "anything" | - * empty - * - * the anything in quotes can contain " escaped with \ - * - * Return the char that caused failure on fail (EOF or \n). - */ -static int -read_double( FILE *fp, const char whitemap[256], const char sepmap[256], - int lineno, int colno, double *out, gboolean fail ) -{ - int ch; - - /* The fscanf() may change this ... but all other cases need a zero. - */ - *out = 0; - - ch = skip_white( fp, whitemap ); - if( ch == EOF || - ch == '\n' ) - return( ch ); - - if( ch == '"' ) { - (void) vips__fgetc( fp ); - (void) skip_to_quote( fp ); - (void) vips__fgetc( fp ); - } - else if( !sepmap[ch] && - fscanf( fp, "%lf", out ) != 1 ) { - /* Only a warning, since (for example) exported spreadsheets - * will often have text or date fields. - */ - g_warning( _( "error parsing number, line %d, column %d" ), - lineno, colno ); - if( fail ) - return( EOF ); - - /* Step over the bad data to the next separator. - */ - (void) skip_to_sep( fp, sepmap ); - } - - /* Don't need to check result, we have read a field successfully. - */ - ch = skip_white( fp, whitemap ); - - /* If it's a separator, we have to step over it. - */ - if( ch != EOF && - sepmap[ch] ) - (void) vips__fgetc( fp ); - - return( 0 ); -} - -static int -read_csv( FILE *fp, VipsImage *out, - int skip, - int lines, - const char *whitespace, const char *separator, - gboolean read_image, - gboolean fail ) -{ - int i; - char whitemap[256]; - char sepmap[256]; - const char *p; - fpos_t pos; - int columns; - int ch; - double d; - double *buf; - int y; - - /* Make our char maps. - */ - for( i = 0; i < 256; i++ ) { - whitemap[i] = 0; - sepmap[i] = 0; - } - for( p = whitespace; *p; p++ ) - whitemap[(int) *p] = 1; - for( p = separator; *p; p++ ) - sepmap[(int) *p] = 1; - - /* Skip first few lines. - */ - for( i = 0; i < skip; i++ ) - if( !skip_line( fp ) ) { - vips_error( "csv2vips", - "%s", _( "end of file while skipping start" ) ); - return( -1 ); - } - - /* Parse the first line to get number of columns. Only bother checking - * fgetpos() the first time we use it: assume it's working after this. - */ - if( fgetpos( fp, &pos ) ) { - vips_error_system( errno, "csv2vips", - "%s", _( "unable to seek" ) ); - return( -1 ); - } - for( columns = 0; - (ch = read_double( fp, whitemap, sepmap, - skip + 1, columns + 1, &d, fail )) == 0; - columns++ ) - ; - (void) fsetpos( fp, &pos ); - - if( columns == 0 ) { - vips_error( "csv2vips", "%s", _( "empty line" ) ); - return( -1 ); - } - - /* If lines is -1, we have to scan the whole file to get the - * number of lines out. - */ - if( lines == -1 ) { - (void) fgetpos( fp, &pos ); - for( lines = 0; skip_line( fp ); lines++ ) - ; - (void) fsetpos( fp, &pos ); - } - - vips_image_pipelinev( out, VIPS_DEMAND_STYLE_THINSTRIP, NULL ); - vips_image_init_fields( out, - columns, lines, 1, - VIPS_FORMAT_DOUBLE, - VIPS_CODING_NONE, VIPS_INTERPRETATION_B_W, 1.0, 1.0 ); - - /* Just reading the header? We are done. - */ - if( !read_image ) - return( 0 ); - - if( !(buf = VIPS_ARRAY( out, - VIPS_IMAGE_N_ELEMENTS( out ), double )) ) - return( -1 ); - - for( y = 0; y < lines; y++ ) { - int x; - - for( x = 0; x < columns; x++ ) { - int lineno = y + skip + 1; - int colno = x + 1; - - ch = read_double( fp, whitemap, sepmap, - lineno, colno, &d, fail ); - if( ch == EOF ) { - vips_error( "csv2vips", - _( "unexpected EOF, line %d col %d" ), - lineno, colno ); - return( -1 ); - } - else if( ch == '\n' ) { - vips_error( "csv2vips", - _( "unexpected EOL, line %d col %d" ), - lineno, colno ); - return( -1 ); - } - else if( ch ) - /* Parse error. - */ - return( -1 ); - - buf[x] = d; - } - - if( vips_image_write_line( out, y, (VipsPel *) buf ) ) - return( -1 ); - - /* Skip over the '\n' to the next line. - */ - skip_line( fp ); - } - - return( 0 ); -} - -int -vips__csv_read( const char *filename, VipsImage *out, - int skip, int lines, const char *whitespace, const char *separator, - gboolean fail ) -{ - FILE *fp; - - if( !(fp = vips__file_open_read( filename, NULL, TRUE )) ) - return( -1 ); - if( read_csv( fp, out, - skip, lines, whitespace, separator, TRUE, fail ) ) { - fclose( fp ); - return( -1 ); - } - fclose( fp ); - - return( 0 ); -} - -int -vips__csv_read_header( const char *filename, VipsImage *out, - int skip, int lines, const char *whitespace, const char *separator, - gboolean fail ) -{ - FILE *fp; - - if( !(fp = vips__file_open_read( filename, NULL, TRUE )) ) - return( -1 ); - if( read_csv( fp, out, - skip, lines, whitespace, separator, FALSE, fail ) ) { - fclose( fp ); - return( -1 ); - } - fclose( fp ); - - return( 0 ); -} - -const char *vips__foreign_csv_suffs[] = { ".csv", NULL }; - -#define PRINT_INT( TYPE ) fprintf( fp, "%d", *((TYPE*)p) ); -#define PRINT_FLOAT( TYPE ) fprintf( fp, "%g", *((TYPE*)p) ); -#define PRINT_COMPLEX( TYPE ) fprintf( fp, "(%g, %g)", \ - ((TYPE*)p)[0], ((TYPE*)p)[1] ); - -static int -vips2csv( VipsImage *in, FILE *fp, const char *sep ) -{ - int w = VIPS_IMAGE_N_ELEMENTS( in ); - int es = VIPS_IMAGE_SIZEOF_ELEMENT( in ); - - int x, y; - VipsPel *p; - - p = in->data; - for( y = 0; y < in->Ysize; y++ ) { - for( x = 0; x < w; x++ ) { - if( x > 0 ) - fprintf( fp, "%s", sep ); - - switch( in->BandFmt ) { - case VIPS_FORMAT_UCHAR: - PRINT_INT( unsigned char ); break; - case VIPS_FORMAT_CHAR: - PRINT_INT( char ); break; - case VIPS_FORMAT_USHORT: - PRINT_INT( unsigned short ); break; - case VIPS_FORMAT_SHORT: - PRINT_INT( short ); break; - case VIPS_FORMAT_UINT: - PRINT_INT( unsigned int ); break; - case VIPS_FORMAT_INT: - PRINT_INT( int ); break; - case VIPS_FORMAT_FLOAT: - PRINT_FLOAT( float ); break; - case VIPS_FORMAT_DOUBLE: - PRINT_FLOAT( double ); break; - case VIPS_FORMAT_COMPLEX: - PRINT_COMPLEX( float ); break; - case VIPS_FORMAT_DPCOMPLEX: - PRINT_COMPLEX( double ); break; - - default: - g_assert_not_reached(); - } - - p += es; - } - - fprintf( fp, "\n" ); - } - - return( 0 ); -} - -int -vips__csv_write( VipsImage *in, const char *filename, const char *separator ) -{ - FILE *fp; - VipsImage *memory; - - if( vips_check_mono( "vips2csv", in ) || - vips_check_uncoded( "vips2csv", in ) || - !(memory = vips_image_copy_memory( in )) ) - return( -1 ); - - if( !(fp = vips__file_open_write( filename, TRUE )) ) { - VIPS_UNREF( memory ); - return( -1 ); - } - - if( vips2csv( memory, fp, separator ) ) { - fclose( fp ); - VIPS_UNREF( memory ); - return( -1 ); - } - fclose( fp ); - VIPS_UNREF( memory ); - - return( 0 ); -} - /* Read to non-whitespace, or buffer overflow. */ static int diff --git a/libvips/foreign/csvload.c b/libvips/foreign/csvload.c index 41825706..2505e449 100644 --- a/libvips/foreign/csvload.c +++ b/libvips/foreign/csvload.c @@ -2,6 +2,8 @@ * * 5/12/11 * - from csvload.c + * 21/2/20 + * - rewrite for new source API */ /* @@ -43,6 +45,7 @@ #include #include #include +#include #include #include @@ -50,18 +53,47 @@ #include "pforeign.h" +/* The largest item we can read. It only needs to be big enough for a double. + */ +#define MAX_ITEM_SIZE (256) + typedef struct _VipsForeignLoadCsv { VipsForeignLoad parent_object; - /* Filename for load. + /* Set by subclasses. */ - char *filename; + VipsSource *source; + /* Buffered source. + */ + VipsSbuf *sbuf; + + /* Load options. + */ int skip; int lines; const char *whitespace; const char *separator; + /* Current position in file for error messages. + */ + int lineno; + int colno; + + /* Our whitespace and separator strings turned into LUTs. + */ + char whitemap[256]; + char sepmap[256]; + + /* Fetch items into this buffer. It just needs to be large enough for + * a double. + */ + char item[MAX_ITEM_SIZE]; + + /* A line of pixels. + */ + double *linebuf; + } VipsForeignLoadCsv; typedef VipsForeignLoadClass VipsForeignLoadCsvClass; @@ -69,31 +101,280 @@ typedef VipsForeignLoadClass VipsForeignLoadCsvClass; G_DEFINE_TYPE( VipsForeignLoadCsv, vips_foreign_load_csv, VIPS_TYPE_FOREIGN_LOAD ); -static VipsForeignFlags -vips_foreign_load_csv_get_flags_filename( const char *filename ) +static void +vips_foreign_load_csv_dispose( GObject *gobject ) { + VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) gobject; + + VIPS_UNREF( csv->source ); + VIPS_UNREF( csv->sbuf ); + VIPS_FREE( csv->linebuf ); + + G_OBJECT_CLASS( vips_foreign_load_csv_parent_class )-> + dispose( gobject ); +} + +static int +vips_foreign_load_csv_build( VipsObject *object ) +{ + VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) object; + + int i; + const char *p; + + if( !(csv->sbuf = vips_sbuf_new_from_source( csv->source )) ) + return( -1 ); + + /* Make our char maps. + */ + for( i = 0; i < 256; i++ ) { + csv->whitemap[i] = 0; + csv->sepmap[i] = 0; + } + for( p = csv->whitespace; *p; p++ ) + csv->whitemap[(int) *p] = 1; + for( p = csv->separator; *p; p++ ) + csv->sepmap[(int) *p] = 1; + + /* \n must not be in the maps or we'll get very confused. + */ + csv->sepmap[(int) '\n'] = 0; + csv->whitemap[(int) '\n'] = 0; + + if( VIPS_OBJECT_CLASS( vips_foreign_load_csv_parent_class )-> + build( object ) ) + return( -1 ); + return( 0 ); } static VipsForeignFlags vips_foreign_load_csv_get_flags( VipsForeignLoad *load ) { - VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) load; + return( 0 ); +} - return( vips_foreign_load_csv_get_flags_filename( csv->filename ) ); +/* Skip to the start of the next block of non-whitespace. + * + * Result: !white, \n, EOF + */ +static int +vips_foreign_load_csv_skip_white( VipsForeignLoadCsv *csv ) +{ + int ch; + + do { + ch = VIPS_SBUF_GETC( csv->sbuf ); + } while( ch != EOF && + ch != '\n' && + csv->whitemap[ch] ); + + VIPS_SBUF_UNGETC( csv->sbuf ); + + return( ch ); +} + +/* We have just seen " (open quotes). Skip to just after the matching close + * quotes. + * + * If there is no matching close quotes before the end of the line, don't + * skip to the next line. + * + * Result: ", \n, EOF + */ +static int +vips_foreign_load_csv_skip_quoted( VipsForeignLoadCsv *csv ) +{ + int ch; + + do { + ch = VIPS_SBUF_GETC( csv->sbuf ); + + /* Ignore \" (actually \anything) in strings. + */ + if( ch == '\\' ) + ch = VIPS_SBUF_GETC( csv->sbuf ); + else if( ch == '"' ) + break; + } while( ch != EOF && + ch != '\n' ); + + if( ch == '\n' ) + VIPS_SBUF_UNGETC( csv->sbuf ); + + return( ch ); +} + +/* Fetch the next item (not whitespace, separator or \n), as a string. The + * returned string is valid until the next call to fetch item. NULL for EOF. + */ +static const char * +vips_foreign_load_csv_fetch_item( VipsForeignLoadCsv *csv ) +{ + int write_point; + int space_remaining; + int ch; + + /* -1 so there's space for the \0 terminator. + */ + space_remaining = MAX_ITEM_SIZE - 1; + write_point = 0; + + while( (ch = VIPS_SBUF_GETC( csv->sbuf )) != -1 && + ch != '\n' && + !csv->whitemap[ch] && + !csv->sepmap[ch] && + space_remaining > 0 ) { + csv->item[write_point] = ch; + write_point += 1; + space_remaining -= 1; + } + csv->item[write_point] = '\0'; + + /* If we hit EOF immediately, return EOF. + */ + if( ch == -1 && + write_point == 0 ) + return( NULL ); + + /* If we filled the item buffer without seeing the end of the item, + * read up to the item end. + */ + while( ch != -1 && + ch != '\n' && + !csv->whitemap[ch] && + !csv->sepmap[ch] ) + ch = VIPS_SBUF_GETC( csv->sbuf ); + + /* We've (probably) read the end of item character. Push it bakc. + */ + if( ch == '\n' || + csv->whitemap[ch] || + csv->sepmap[ch] ) + VIPS_SBUF_UNGETC( csv->sbuf ); + + return( csv->item ); +} + +/* Read a single item. The syntax is: + * + * element : + * whitespace* item whitespace* [EOF|EOL|separator] + * + * item : + * double | + * "anything" | + * empty + * + * the anything in quotes can contain " escaped with \, and can contain + * separator and whitespace characters. + * + * Result: sep, \n, EOF + */ +static int +vips_foreign_load_csv_read_double( VipsForeignLoadCsv *csv, double *out ) +{ + int ch; + + /* The strtod() may change this ... but all other cases need a zero. + */ + *out = 0; + + ch = vips_foreign_load_csv_skip_white( csv ); + if( ch == EOF || + ch == '\n' ) + return( ch ); + + if( ch == '"' ) { + (void) VIPS_SBUF_GETC( csv->sbuf ); + ch = vips_foreign_load_csv_skip_quoted( csv ); + } + else if( !csv->sepmap[ch] ) { + const char *item; + + item = vips_foreign_load_csv_fetch_item( csv ); + if( !item ) + return( EOF ); + *out = g_ascii_strtod( item, NULL ); + if( errno ) + /* Only a warning, since (for example) exported + * spreadsheets will often have text or date fields. + */ + g_warning( _( "bad number, line %d, column %d" ), + csv->lineno, csv->colno ); + } + + ch = vips_foreign_load_csv_skip_white( csv ); + if( ch == EOF || + ch == '\n' ) + return( ch ); + + /* If it's a separator, we have to step over it. + */ + if( csv->sepmap[ch] ) { + (void) VIPS_SBUF_GETC( csv->sbuf ); + csv->colno += 1; + } + + return( ch ); } static int vips_foreign_load_csv_header( VipsForeignLoad *load ) { VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) load; + VipsObjectClass *class = VIPS_OBJECT_GET_CLASS( load ); - if( vips__csv_read_header( csv->filename, load->out, - csv->skip, csv->lines, csv->whitespace, csv->separator, - load->fail ) ) + int i; + double value; + int ch; + int width; + int height; + + /* Rewind. + */ + vips_sbuf_unbuffer( csv->sbuf ); + if( vips_source_rewind( csv->source ) ) return( -1 ); - VIPS_SETSTR( load->out->filename, csv->filename ); + /* Skip the first few lines. + */ + for( i = 0; i < csv->skip; i++ ) + if( !vips_sbuf_get_line( csv->sbuf ) ) { + vips_error( class->nickname, + "%s", _( "unexpected end of file" ) ); + return( -1 ); + } + + /* Parse the first line to get the number of columns. + */ + csv->colno = 1; + csv->lineno = csv->skip; + do { + ch = vips_foreign_load_csv_read_double( csv, &value ); + } while( ch != '\n' && + ch != EOF ); + width = csv->colno; + + if( !(csv->linebuf = VIPS_ARRAY( NULL, width, double )) ) + return( -1 ); + + /* If @lines is -1, we must scan the whole file to get the height. + */ + if( csv->lines == -1 ) + for( height = 0; vips_sbuf_get_line( csv->sbuf ); height++ ) + ; + else + height = csv->lines; + + vips_image_pipelinev( load->out, VIPS_DEMAND_STYLE_THINSTRIP, NULL ); + vips_image_init_fields( load->out, + width, height, 1, + VIPS_FORMAT_DOUBLE, + VIPS_CODING_NONE, VIPS_INTERPRETATION_B_W, 1.0, 1.0 ); + + VIPS_SETSTR( load->out->filename, + vips_connection_filename( VIPS_CONNECTION( csv->source ) ) ); return( 0 ); } @@ -102,12 +383,70 @@ static int vips_foreign_load_csv_load( VipsForeignLoad *load ) { VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) load; + VipsObjectClass *class = VIPS_OBJECT_GET_CLASS( load ); - if( vips__csv_read( csv->filename, load->real, - csv->skip, csv->lines, csv->whitespace, csv->separator, - load->fail ) ) + int i; + int x, y; + int ch; + + /* Rewind. + */ + vips_sbuf_unbuffer( csv->sbuf ); + if( vips_source_rewind( csv->source ) ) return( -1 ); + /* Skip the first few lines. + */ + for( i = 0; i < csv->skip; i++ ) + if( !vips_sbuf_get_line( csv->sbuf ) ) { + vips_error( class->nickname, + "%s", _( "unexpected end of file" ) ); + return( -1 ); + } + + vips_image_pipelinev( load->real, VIPS_DEMAND_STYLE_THINSTRIP, NULL ); + vips_image_init_fields( load->real, + load->out->Xsize, load->out->Ysize, 1, + VIPS_FORMAT_DOUBLE, + VIPS_CODING_NONE, VIPS_INTERPRETATION_B_W, 1.0, 1.0 ); + + csv->lineno = csv->skip; + for( y = 0; y < load->real->Ysize; y++ ) { + csv->colno = 1; + + for( x = 0; x < load->real->Xsize; x++ ) { + double value; + + ch = vips_foreign_load_csv_read_double( csv, &value ); + if( ch == EOF ) { + vips_error( class->nickname, + "%s", _( "unexpected end of file" ) ); + return( -1 ); + } + if( ch == '\n' && + x != load->real->Xsize - 1 ) { + vips_error( class->nickname, + _( "line %d has only %d columns" ), + csv->lineno, csv->colno ); + if( load->fail ) + return( -1 ); + } + + csv->linebuf[x] = value; + } + + /* Step over the line separator. + */ + if( ch == '\n' ) { + (void) VIPS_SBUF_GETC( csv->sbuf ); + csv->lineno += 1; + } + + if( vips_image_write_line( load->real, y, + (VipsPel *) csv->linebuf ) ) + return( -1 ); + } + return( 0 ); } @@ -116,30 +455,20 @@ vips_foreign_load_csv_class_init( VipsForeignLoadCsvClass *class ) { GObjectClass *gobject_class = G_OBJECT_CLASS( class ); VipsObjectClass *object_class = (VipsObjectClass *) class; - VipsForeignClass *foreign_class = (VipsForeignClass *) class; VipsForeignLoadClass *load_class = (VipsForeignLoadClass *) class; + gobject_class->dispose = vips_foreign_load_csv_dispose; gobject_class->set_property = vips_object_set_property; gobject_class->get_property = vips_object_get_property; - object_class->nickname = "csvload"; - object_class->description = _( "load csv from file" ); + object_class->nickname = "csvload_base"; + object_class->description = _( "load csv" ); + object_class->build = vips_foreign_load_csv_build; - foreign_class->suffs = vips__foreign_csv_suffs; - - load_class->get_flags_filename = - vips_foreign_load_csv_get_flags_filename; load_class->get_flags = vips_foreign_load_csv_get_flags; load_class->header = vips_foreign_load_csv_header; load_class->load = vips_foreign_load_csv_load; - VIPS_ARG_STRING( class, "filename", 1, - _( "Filename" ), - _( "Filename to load from" ), - VIPS_ARGUMENT_REQUIRED_INPUT, - G_STRUCT_OFFSET( VipsForeignLoadCsv, filename ), - NULL ); - VIPS_ARG_INT( class, "skip", 20, _( "Skip" ), _( "Skip this many lines at the start of the file" ), @@ -177,6 +506,138 @@ vips_foreign_load_csv_init( VipsForeignLoadCsv *csv ) csv->separator = g_strdup( ";,\t" ); } +typedef struct _VipsForeignLoadCsvFile { + VipsForeignLoadCsv parent_object; + + /* Filename for load. + */ + char *filename; + +} VipsForeignLoadCsvFile; + +typedef VipsForeignLoadCsvClass VipsForeignLoadCsvFileClass; + +G_DEFINE_TYPE( VipsForeignLoadCsvFile, vips_foreign_load_csv_file, + vips_foreign_load_csv_get_type() ); + +static VipsForeignFlags +vips_foreign_load_csv_file_get_flags_filename( const char *filename ) +{ + return( 0 ); +} + +static int +vips_foreign_load_csv_file_build( VipsObject *object ) +{ + VipsForeignLoadCsvFile *file = (VipsForeignLoadCsvFile *) object; + VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) object; + + if( file->filename ) + if( !(csv->source = + vips_source_new_from_file( file->filename )) ) + return( -1 ); + + if( VIPS_OBJECT_CLASS( vips_foreign_load_csv_file_parent_class )-> + build( object ) ) + return( -1 ); + + return( 0 ); +} + +static const char *vips_foreign_load_csv_suffs[] = { + ".csv", + NULL +}; + +static void +vips_foreign_load_csv_file_class_init( VipsForeignLoadCsvFileClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *object_class = (VipsObjectClass *) class; + VipsForeignClass *foreign_class = (VipsForeignClass *) class; + VipsForeignLoadClass *load_class = (VipsForeignLoadClass *) class; + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + object_class->nickname = "csvload"; + object_class->build = vips_foreign_load_csv_file_build; + + foreign_class->suffs = vips_foreign_load_csv_suffs; + + load_class->get_flags_filename = + vips_foreign_load_csv_file_get_flags_filename; + + VIPS_ARG_STRING( class, "filename", 1, + _( "Filename" ), + _( "Filename to load from" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadCsvFile, filename ), + NULL ); + +} + +static void +vips_foreign_load_csv_file_init( VipsForeignLoadCsvFile *file ) +{ +} + +typedef struct _VipsForeignLoadCsvSource { + VipsForeignLoadCsv parent_object; + + VipsSource *source; + +} VipsForeignLoadCsvSource; + +typedef VipsForeignLoadCsvClass VipsForeignLoadCsvSourceClass; + +G_DEFINE_TYPE( VipsForeignLoadCsvSource, vips_foreign_load_csv_source, + vips_foreign_load_csv_get_type() ); + +static int +vips_foreign_load_csv_source_build( VipsObject *object ) +{ + VipsForeignLoadCsvSource *source = (VipsForeignLoadCsvSource *) object; + VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) object; + + if( csv->source ) { + csv->source = source->source; + g_object_ref( csv->source ); + } + + if( VIPS_OBJECT_CLASS( vips_foreign_load_csv_source_parent_class )-> + build( object ) ) + return( -1 ); + + return( 0 ); +} + +static void +vips_foreign_load_csv_source_class_init( VipsForeignLoadCsvFileClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *object_class = (VipsObjectClass *) class; + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + object_class->nickname = "csvload_source"; + object_class->build = vips_foreign_load_csv_source_build; + + VIPS_ARG_OBJECT( class, "source", 1, + _( "Source" ), + _( "Source to load from" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadCsvSource, source ), + VIPS_TYPE_SOURCE ); + +} + +static void +vips_foreign_load_csv_source_init( VipsForeignLoadCsvSource *source ) +{ +} + /** * vips_csvload: * @filename: file to load @@ -200,10 +661,6 @@ vips_foreign_load_csv_init( VipsForeignLoadCsv *csv ) * You can use a backslash (\) within the quotes to escape special characters, * such as quote marks. * - * The reader is deliberately rather fussy: it will fail if there are any - * short lines, or if the file is too short. It will ignore lines that are - * too long. - * * @skip sets the number of lines to skip at the start of the file. * Default zero. * @@ -236,4 +693,39 @@ vips_csvload( const char *filename, VipsImage **out, ... ) return( result ); } +/** + * vips_csvload_source: + * @source: source to load + * @out: (out): output image + * @...: %NULL-terminated list of optional named arguments + * + * Optional arguments: + * + * * @skip: skip this many lines at start of file + * * @lines: read this many lines from file + * * @whitespace: set of whitespace characters + * * @separator: set of separator characters + * * @fail: %gboolean, fail on errors + * + * Exactly as vips_csvload(), but read from a source. + * + * See also: vips_csvload(). + * + * Returns: 0 on success, -1 on error. + */ +int +vips_csvload_source( VipsSource *source, VipsImage **out, ... ) +{ + va_list ap; + int result; + + va_start( ap, out ); + result = vips_call_split( "csvload_source", ap, source, out ); + va_end( ap ); + + return( result ); +} + + + diff --git a/libvips/foreign/csvsave.c b/libvips/foreign/csvsave.c index 633f9ac9..4bb88fe9 100644 --- a/libvips/foreign/csvsave.c +++ b/libvips/foreign/csvsave.c @@ -2,6 +2,8 @@ * * 2/12/11 * - wrap a class around the csv writer + * 21/2/20 + * - rewrite for the VipsTarget API */ /* @@ -52,9 +54,7 @@ typedef struct _VipsForeignSaveCsv { VipsForeignSave parent_object; - /* Filename for save. - */ - char *filename; + VipsTarget *target; const char *separator; } VipsForeignSaveCsv; @@ -64,17 +64,119 @@ typedef VipsForeignSaveClass VipsForeignSaveCsvClass; G_DEFINE_TYPE( VipsForeignSaveCsv, vips_foreign_save_csv, VIPS_TYPE_FOREIGN_SAVE ); +static void +vips_foreign_save_csv_dispose( GObject *gobject ) +{ + VipsForeignSaveCsv *csv = (VipsForeignSaveCsv *) gobject; + + if( csv->target ) + vips_target_finish( csv->target ); + VIPS_UNREF( csv->target ); + + G_OBJECT_CLASS( vips_foreign_save_csv_parent_class )-> + dispose( gobject ); +} + +#define PRINT_INT( TYPE ) { \ + TYPE *pt = (TYPE *) p; \ + \ + for( x = 0; x < image->Xsize; x++ ) { \ + if( x > 0 ) \ + vips_target_writes( csv->target, csv->separator ); \ + vips_target_writef( csv->target, "%d", pt[x] ); \ + } \ +} + +#define PRINT_FLOAT( TYPE ) { \ + TYPE *pt = (TYPE *) p; \ + char buf[G_ASCII_DTOSTR_BUF_SIZE]; \ + \ + for( x = 0; x < image->Xsize; x++ ) { \ + if( x > 0 ) \ + vips_target_writes( csv->target, csv->separator ); \ + g_ascii_dtostr( buf, G_ASCII_DTOSTR_BUF_SIZE, pt[x] ); \ + vips_target_writes( csv->target, buf ); \ + } \ +} + +#define PRINT_COMPLEX( TYPE ) { \ + TYPE *pt = (TYPE *) p; \ + char buf[G_ASCII_DTOSTR_BUF_SIZE]; \ + \ + for( x = 0; x < image->Xsize; x++ ) { \ + if( x > 0 ) \ + vips_target_writes( csv->target, csv->separator ); \ + VIPS_TARGET_PUTC( csv->target, '(' ); \ + g_ascii_dtostr( buf, G_ASCII_DTOSTR_BUF_SIZE, pt[0] ); \ + vips_target_writes( csv->target, buf ); \ + VIPS_TARGET_PUTC( csv->target, ',' ); \ + g_ascii_dtostr( buf, G_ASCII_DTOSTR_BUF_SIZE, pt[1] ); \ + vips_target_writes( csv->target, buf ); \ + VIPS_TARGET_PUTC( csv->target, ')' ); \ + pt += 2; \ + } \ +} + +static int +vips_foreign_save_csv_block( VipsRegion *region, VipsRect *area, void *a ) +{ + VipsForeignSaveCsv *csv = (VipsForeignSaveCsv *) a; + VipsImage *image = region->im; + + int x, y; + + for( y = 0; y < area->height; y++ ) { + VipsPel *p = VIPS_REGION_ADDR( region, 0, area->top + y ); + + switch( image->BandFmt ) { + case VIPS_FORMAT_UCHAR: + PRINT_INT( unsigned char ); break; + case VIPS_FORMAT_CHAR: + PRINT_INT( char ); break; + case VIPS_FORMAT_USHORT: + PRINT_INT( unsigned short ); break; + case VIPS_FORMAT_SHORT: + PRINT_INT( short ); break; + case VIPS_FORMAT_UINT: + PRINT_INT( unsigned int ); break; + case VIPS_FORMAT_INT: + PRINT_INT( int ); break; + case VIPS_FORMAT_FLOAT: + PRINT_FLOAT( float ); break; + case VIPS_FORMAT_DOUBLE: + PRINT_FLOAT( double ); break; + case VIPS_FORMAT_COMPLEX: + PRINT_COMPLEX( float ); break; + case VIPS_FORMAT_DPCOMPLEX: + PRINT_COMPLEX( double ); break; + + default: + g_assert_not_reached(); + } + + if( vips_target_writes( csv->target, "\n" ) ) + return( -1 ); + } + + return( 0 ); +} + static int vips_foreign_save_csv_build( VipsObject *object ) { VipsForeignSave *save = (VipsForeignSave *) object; VipsForeignSaveCsv *csv = (VipsForeignSaveCsv *) object; + VipsObjectClass *class = VIPS_OBJECT_GET_CLASS( object ); if( VIPS_OBJECT_CLASS( vips_foreign_save_csv_parent_class )-> build( object ) ) return( -1 ); - if( vips__csv_write( save->ready, csv->filename, csv->separator ) ) + if( vips_check_mono( class->nickname, save->ready ) || + vips_check_uncoded( class->nickname, save->ready ) ) + return( -1 ); + + if( vips_sink_disc( save->ready, vips_foreign_save_csv_block, csv ) ) return( -1 ); return( 0 ); @@ -85,27 +187,18 @@ vips_foreign_save_csv_class_init( VipsForeignSaveCsvClass *class ) { GObjectClass *gobject_class = G_OBJECT_CLASS( class ); VipsObjectClass *object_class = (VipsObjectClass *) class; - VipsForeignClass *foreign_class = (VipsForeignClass *) class; VipsForeignSaveClass *save_class = (VipsForeignSaveClass *) class; + gobject_class->dispose = vips_foreign_save_csv_dispose; gobject_class->set_property = vips_object_set_property; gobject_class->get_property = vips_object_get_property; - object_class->nickname = "csvsave"; - object_class->description = _( "save image to csv file" ); + object_class->nickname = "csvsave_base"; + object_class->description = _( "save image to csv" ); object_class->build = vips_foreign_save_csv_build; - foreign_class->suffs = vips__foreign_csv_suffs; - save_class->saveable = VIPS_SAVEABLE_MONO; - VIPS_ARG_STRING( class, "filename", 1, - _( "Filename" ), - _( "Filename to save to" ), - VIPS_ARGUMENT_REQUIRED_INPUT, - G_STRUCT_OFFSET( VipsForeignSaveCsv, filename ), - NULL ); - VIPS_ARG_STRING( class, "separator", 13, _( "Separator" ), _( "Separator characters" ), @@ -120,6 +213,117 @@ vips_foreign_save_csv_init( VipsForeignSaveCsv *csv ) csv->separator = g_strdup( "\t" ); } +typedef struct _VipsForeignSaveCsvFile { + VipsForeignSaveCsv parent_object; + + char *filename; +} VipsForeignSaveCsvFile; + +typedef VipsForeignSaveCsvClass VipsForeignSaveCsvFileClass; + +G_DEFINE_TYPE( VipsForeignSaveCsvFile, vips_foreign_save_csv_file, + vips_foreign_save_csv_get_type() ); + +static int +vips_foreign_save_csv_file_build( VipsObject *object ) +{ + VipsForeignSaveCsv *csv = (VipsForeignSaveCsv *) object; + VipsForeignSaveCsvFile *file = (VipsForeignSaveCsvFile *) object; + + if( file->filename && + !(csv->target = vips_target_new_to_file( file->filename )) ) + return( -1 ); + + return( VIPS_OBJECT_CLASS( vips_foreign_save_csv_file_parent_class )-> + build( object ) ); +} + +static const char *vips_foreign_save_csv_file_suffs[] = { + ".csv", + NULL +}; + +static void +vips_foreign_save_csv_file_class_init( VipsForeignSaveCsvFileClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *object_class = (VipsObjectClass *) class; + VipsForeignClass *foreign_class = (VipsForeignClass *) class; + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + object_class->nickname = "csvsave"; + object_class->build = vips_foreign_save_csv_file_build; + + foreign_class->suffs = vips_foreign_save_csv_file_suffs; + + VIPS_ARG_STRING( class, "filename", 1, + _( "Filename" ), + _( "Filename to save to" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsForeignSaveCsvFile, filename ), + NULL ); + +} + +static void +vips_foreign_save_csv_file_init( VipsForeignSaveCsvFile *file ) +{ +} + +typedef struct _VipsForeignSaveCsvTarget { + VipsForeignSaveCsv parent_object; + + VipsTarget *target; +} VipsForeignSaveCsvTarget; + +typedef VipsForeignSaveCsvClass VipsForeignSaveCsvTargetClass; + +G_DEFINE_TYPE( VipsForeignSaveCsvTarget, vips_foreign_save_csv_target, + vips_foreign_save_csv_get_type() ); + +static int +vips_foreign_save_csv_target_build( VipsObject *object ) +{ + VipsForeignSaveCsv *csv = (VipsForeignSaveCsv *) object; + VipsForeignSaveCsvTarget *target = (VipsForeignSaveCsvTarget *) object; + + if( target->target ) { + csv->target = target->target; + g_object_ref( csv->target ); + } + + return( VIPS_OBJECT_CLASS( vips_foreign_save_csv_target_parent_class )-> + build( object ) ); +} + +static void +vips_foreign_save_csv_target_class_init( VipsForeignSaveCsvTargetClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *object_class = (VipsObjectClass *) class; + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + object_class->nickname = "csvsave_target"; + object_class->build = vips_foreign_save_csv_target_build; + + VIPS_ARG_OBJECT( class, "target", 1, + _( "Target" ), + _( "Target to save to" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsForeignSaveCsvTarget, target ), + VIPS_TYPE_TARGET ); + +} + +static void +vips_foreign_save_csv_target_init( VipsForeignSaveCsvTarget *target ) +{ +} + /** * vips_csvsave: (method) * @in: image to save @@ -155,3 +359,32 @@ vips_csvsave( VipsImage *in, const char *filename, ... ) return( result ); } + +/** + * vips_csvsave_target: (method) + * @in: image to save + * @target: save image to this target + * @...: %NULL-terminated list of optional named arguments + * + * Optional arguments: + * + * * @separator: separator string + * + * As vips_csvsave(), but save to a target. + * + * See also: vips_csvsave(). + * + * Returns: 0 on success, -1 on error. + */ +int +vips_csvsave_target( VipsImage *in, VipsTarget *target, ... ) +{ + va_list ap; + int result; + + va_start( ap, target ); + result = vips_call_split( "csvsave_target", ap, in, target ); + va_end( ap ); + + return( result ); +} diff --git a/libvips/foreign/foreign.c b/libvips/foreign/foreign.c index 86632973..56826faf 100644 --- a/libvips/foreign/foreign.c +++ b/libvips/foreign/foreign.c @@ -2034,8 +2034,10 @@ vips_foreign_operation_init( void ) extern GType vips_foreign_save_png_buffer_get_type( void ); extern GType vips_foreign_save_png_target_get_type( void ); - extern GType vips_foreign_load_csv_get_type( void ); - extern GType vips_foreign_save_csv_get_type( void ); + extern GType vips_foreign_load_csv_file_get_type( void ); + extern GType vips_foreign_load_csv_source_get_type( void ); + extern GType vips_foreign_save_csv_file_get_type( void ); + extern GType vips_foreign_save_csv_target_get_type( void ); extern GType vips_foreign_load_matrix_get_type( void ); extern GType vips_foreign_save_matrix_get_type( void ); @@ -2109,8 +2111,10 @@ vips_foreign_operation_init( void ) extern GType vips_foreign_load_gif_buffer_get_type( void ); extern GType vips_foreign_load_gif_source_get_type( void ); - vips_foreign_load_csv_get_type(); - vips_foreign_save_csv_get_type(); + vips_foreign_load_csv_file_get_type(); + vips_foreign_load_csv_source_get_type(); + vips_foreign_save_csv_file_get_type(); + vips_foreign_save_csv_target_get_type(); vips_foreign_load_matrix_get_type(); vips_foreign_save_matrix_get_type(); vips_foreign_print_matrix_get_type(); diff --git a/libvips/foreign/ppmsave.c b/libvips/foreign/ppmsave.c index 25f59a29..0f733777 100644 --- a/libvips/foreign/ppmsave.c +++ b/libvips/foreign/ppmsave.c @@ -186,10 +186,11 @@ vips_foreign_save_ppm_block( VipsRegion *region, VipsRect *area, void *a ) { VipsForeignSavePpm *ppm = (VipsForeignSavePpm *) a; VipsImage *image = region->im; - int i; - for( i = 0; i < area->height; i++ ) { - VipsPel *p = VIPS_REGION_ADDR( region, 0, area->top + i ); + int y; + + for( y = 0; y < area->height; y++ ) { + VipsPel *p = VIPS_REGION_ADDR( region, 0, area->top + y ); if( ppm->fn( ppm, image, p ) ) return( -1 ); @@ -450,7 +451,6 @@ vips_foreign_save_ppm_file_init( VipsForeignSavePpmFile *file ) { } - #endif /*HAVE_PPM*/ /** diff --git a/libvips/include/vips/foreign.h b/libvips/include/vips/foreign.h index da9a82a2..52f7a344 100644 --- a/libvips/include/vips/foreign.h +++ b/libvips/include/vips/foreign.h @@ -524,8 +524,12 @@ int vips_rawsave_fd( VipsImage *in, int fd, ... ) int vips_csvload( const char *filename, VipsImage **out, ... ) __attribute__((sentinel)); +int vips_csvload_source( VipsSource *source, VipsImage **out, ... ) + __attribute__((sentinel)); int vips_csvsave( VipsImage *in, const char *filename, ... ) __attribute__((sentinel)); +int vips_csvsave_target( VipsImage *in, VipsTarget *target, ... ) + __attribute__((sentinel)); int vips_matrixload( const char *filename, VipsImage **out, ... ) __attribute__((sentinel)); diff --git a/libvips/iofuncs/sbuf.c b/libvips/iofuncs/sbuf.c index 14134ca5..1a366000 100644 --- a/libvips/iofuncs/sbuf.c +++ b/libvips/iofuncs/sbuf.c @@ -115,6 +115,8 @@ vips_sbuf_new_from_source( VipsSource *source ) { VipsSbuf *sbuf; + g_assert( source ); + sbuf = VIPS_SBUF( g_object_new( VIPS_TYPE_SBUF, "input", source, NULL ) ); @@ -374,8 +376,7 @@ vips_sbuf_get_line( VipsSbuf *sbuf ) */ if( write_point > 0 && sbuf->line[write_point - 1] == '\r' ) - sbuf->line[write_point - 1] = '\0'; - + sbuf->line[write_point - 1] = '\0'; /* If we filled the output line without seeing \n, keep going to the * next \n. */