From 3ee442ca6af7d5a10bdb2fc205d7adfad0347661 Mon Sep 17 00:00:00 2001 From: John Cupitt Date: Fri, 16 Dec 2011 15:24:35 +0000 Subject: [PATCH] move csv load/save to new style --- TODO | 10 +- libvips/conversion/extract.c | 2 +- libvips/foreign/Makefile.am | 4 + libvips/foreign/csv.c | 436 +++++++++++++++++++++++++++++++++ libvips/foreign/csv.h | 51 ++++ libvips/foreign/csvload.c | 227 +++++++++++++++++ libvips/foreign/csvsave.c | 172 +++++++++++++ libvips/foreign/foreign.c | 16 ++ libvips/format/im_csv2vips.c | 363 +-------------------------- libvips/format/im_vips2csv.c | 115 +-------- libvips/include/vips/foreign.h | 7 + libvips/iofuncs/enumtypes.c | 1 + 12 files changed, 938 insertions(+), 466 deletions(-) create mode 100644 libvips/foreign/csv.c create mode 100644 libvips/foreign/csv.h create mode 100644 libvips/foreign/csvload.c create mode 100644 libvips/foreign/csvsave.c diff --git a/TODO b/TODO index b9f32a42..d1f7d3bc 100644 --- a/TODO +++ b/TODO @@ -2,9 +2,15 @@ - test analyze load -- make sure we have a get_flags_filename everywhere + .... appears to be broken, byteswap problem? also, leaks an image + +- test csv load/save + + $ vips im_copy pics/Gugg_coloured.jpg x.csv + Segmentation fault + + some arg mixup? -- threading tests fail? diff --git a/libvips/conversion/extract.c b/libvips/conversion/extract.c index 7a625379..4156f1c3 100644 --- a/libvips/conversion/extract.c +++ b/libvips/conversion/extract.c @@ -66,8 +66,8 @@ */ /* - */ #define VIPS_DEBUG + */ #ifdef HAVE_CONFIG_H #include diff --git a/libvips/foreign/Makefile.am b/libvips/foreign/Makefile.am index 01d61ee2..7a1f2646 100644 --- a/libvips/foreign/Makefile.am +++ b/libvips/foreign/Makefile.am @@ -1,6 +1,10 @@ noinst_LTLIBRARIES = libforeign.la libforeign_la_SOURCES = \ + csv.h \ + csv.c \ + csvload.c \ + csvsave.c \ rawload.c \ rawsave.c \ vipsload.c \ diff --git a/libvips/foreign/csv.c b/libvips/foreign/csv.c new file mode 100644 index 00000000..dd5bbfdf --- /dev/null +++ b/libvips/foreign/csv.c @@ -0,0 +1,436 @@ +/* Read/write csv files. + * + * 19/12/05 JC + * - hacked from ppm reader + * 9/6/06 + * - hacked from im_debugim + * 11/9/06 + * - now distingushes whitespace and separators, so we can have blank + * fields + * 20/9/06 + * - oop, unquoted trailing columns could get missed + * 23/10/06 + * - allow separator to be specified (default "\t", ) + * 17/11/06 + * - oops, was broken + * 17/5/07 + * - added im_csv2vips_header() + * 4/2/10 + * - gtkdoc + * 1/3/10 + * - allow lines that end with EOF + * 23/9/11 + * - allow quoted strings, including escaped quotes + * 16/12/11 + * - rework as a set of fns ready for wrapping as a class + */ + +/* + + This file is part of VIPS. + + VIPS is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + */ + +/* + + These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk + + */ + +#ifdef HAVE_CONFIG_H +#include +#endif /*HAVE_CONFIG_H*/ +#include + +#include +#include +#include +#include +#include + +#include + +#include "csv.h" + +/* Skip to the start of the next line (ie. read until we see a '\n'), return + * zero if we are at EOF. + * + * Files can end with EOF or with \nEOF. Tricky! + */ +static int +skip_line( FILE *fp ) +{ + int ch; + + /* Are we at a delayed EOF? See below. + */ + if( (ch = fgetc( fp )) == EOF ) + return( 0 ); + ungetc( ch, fp ); + + /* If we hit EOF and no \n, wait until the next call to report EOF. + */ + while( (ch = fgetc( fp )) != '\n' && ch != EOF ) + ; + + return( -1 ); +} + +static int +skip_white( FILE *fp, const char whitemap[256] ) +{ + int ch; + + do { + ch = fgetc( fp ); + } while (ch != EOF && ch != '\n' && whitemap[ch] ); + + ungetc( ch, fp ); + + return( ch ); +} + +static int +skip_to_quote( FILE *fp ) +{ + int ch; + + do { + ch = fgetc( fp ); + + /* Ignore \" in strings. + */ + if( ch == '\\' ) + ch = fgetc( fp ); + else if( ch == '"' ) + break; + } while( ch != EOF && ch != '\n' ); + + ungetc( ch, fp ); + + return( ch ); +} + +static int +skip_to_sep( FILE *fp, const char sepmap[256] ) +{ + int ch; + + do { + ch = fgetc( fp ); + } while (ch != EOF && ch != '\n' && !sepmap[ch] ); + + ungetc( ch, fp ); + + return( ch ); +} + +/* Read a single item. Syntax is: + * + * element : + * whitespace* item whitespace* [EOF|EOL|separator] + * + * item : + * double | + * "anything" | + * empty + * + * the anything in quotes can contain " escaped with \ + * + * Return the char that caused failure on fail (EOF or \n). + */ +static int +read_double( FILE *fp, const char whitemap[256], const char sepmap[256], + int lineno, int colno, double *out ) +{ + int ch; + + /* The fscanf() may change this ... but all other cases need a zero. + */ + *out = 0; + + ch = skip_white( fp, whitemap ); + if( ch == EOF || ch == '\n' ) + return( ch ); + + if( ch == '"' ) { + (void) fgetc( fp ); + ch = skip_to_quote( fp ); + ch = fgetc( fp ); + } + else if( !sepmap[ch] && + fscanf( fp, "%lf", out ) != 1 ) { + /* Only a warning, since (for example) exported spreadsheets + * will often have text or date fields. + */ + vips_warn( "csv2vips", + _( "error parsing number, line %d, column %d" ), + lineno, colno ); + + /* Step over the bad data to the next separator. + */ + ch = skip_to_sep( fp, sepmap ); + } + + /* Don't need to check result, we have read a field successfully. + */ + ch = skip_white( fp, whitemap ); + + /* If it's a separator, we have to step over it. + */ + if( ch != EOF && sepmap[ch] ) + (void) fgetc( fp ); + + return( 0 ); +} + +static int +read_csv( FILE *fp, VipsImage *out, + int skip, + int lines, + const char *whitespace, const char *separator, + gboolean read_image ) +{ + int i; + char whitemap[256]; + char sepmap[256]; + const char *p; + fpos_t pos; + int columns; + int ch; + double d; + double *buf; + int y; + + /* Make our char maps. + */ + for( i = 0; i < 256; i++ ) { + whitemap[i] = 0; + sepmap[i] = 0; + } + for( p = whitespace; *p; p++ ) + whitemap[(int) *p] = 1; + for( p = separator; *p; p++ ) + sepmap[(int) *p] = 1; + + /* Skip first few lines. + */ + for( i = 0; i < skip; i++ ) + if( !skip_line( fp ) ) { + vips_error( "csv2vips", + "%s", _( "end of file while skipping start" ) ); + return( -1 ); + } + + /* Parse the first line to get number of columns. Only bother checking + * fgetpos() the first time we use it: assume it's working after this. + */ + if( fgetpos( fp, &pos ) ) { + vips_error_system( errno, "csv2vips", + "%s", _( "unable to seek" ) ); + return( -1 ); + } + for( columns = 0; + (ch = read_double( fp, whitemap, sepmap, + skip + 1, columns + 1, &d )) == 0; + columns++ ) + ; + fsetpos( fp, &pos ); + + if( columns == 0 ) { + vips_error( "csv2vips", "%s", _( "empty line" ) ); + return( -1 ); + } + if( ch == -2 ) + /* Failed to parse a number. + */ + return( -1 ); + + /* If lines is -1, we have to scan the whole file to get the + * number of lines out. + */ + if( lines == -1 ) { + fgetpos( fp, &pos ); + for( lines = 0; skip_line( fp ); lines++ ) + ; + fsetpos( fp, &pos ); + } + + vips_image_init_fields( out, + columns, lines, 1, + VIPS_FORMAT_DOUBLE, + VIPS_CODING_NONE, VIPS_INTERPRETATION_B_W, 1.0, 1.0 ); + vips_demand_hint( out, VIPS_DEMAND_STYLE_THINSTRIP, NULL ); + + /* Just reading the header? We are done. + */ + if( !read_image ) + return( 0 ); + + if( vips_image_wio_output( out ) || + !(buf = VIPS_ARRAY( out, + VIPS_IMAGE_N_ELEMENTS( out ), double )) ) + return( -1 ); + + for( y = 0; y < lines; y++ ) { + int x; + + for( x = 0; x < columns; x++ ) { + ch = read_double( fp, whitemap, sepmap, + y + skip + 1, x + 1, &d ); + if( ch == EOF ) { + vips_error( "csv2vips", + "%s", _( "unexpected end of file" ) ); + return( -1 ); + } + else if( ch == '\n' ) { + vips_error( "csv2vips", + "%s", _( "unexpected end of line" ) ); + return( -1 ); + } + else if( ch ) + /* Parse error. + */ + return( -1 ); + + buf[x] = d; + } + + if( vips_image_write_line( out, y, (PEL *) buf ) ) + return( -1 ); + + /* Skip over the '\n' to the next line. + */ + skip_line( fp ); + } + + return( 0 ); +} + +int +vips__csv_read( const char *filename, VipsImage *out, + int skip, int lines, const char *whitespace, const char *separator ) +{ + FILE *fp; + + if( !(fp = vips__file_open_read( filename, NULL, TRUE )) ) + return( -1 ); + if( read_csv( fp, out, skip, lines, whitespace, separator, TRUE ) ) { + fclose( fp ); + return( -1 ); + } + fclose( fp ); + + return( 0 ); +} + +int +vips__csv_read_header( const char *filename, VipsImage *out, + int skip, int lines, const char *whitespace, const char *separator ) +{ + FILE *fp; + + if( !(fp = vips__file_open_read( filename, NULL, TRUE )) ) + return( -1 ); + if( read_csv( fp, out, skip, lines, whitespace, separator, FALSE ) ) { + fclose( fp ); + return( -1 ); + } + fclose( fp ); + + return( 0 ); +} + +const char *vips__foreign_csv_suffs[] = { ".csv", NULL }; + +#define PRINT_INT( TYPE ) fprintf( fp, "%d", *((TYPE*)p) ); +#define PRINT_FLOAT( TYPE ) fprintf( fp, "%g", *((TYPE*)p) ); +#define PRINT_COMPLEX( TYPE ) fprintf( fp, "(%g, %g)", \ + ((TYPE*)p)[0], ((TYPE*)p)[1] ); + +static int +vips2csv( VipsImage *in, FILE *fp, const char *sep ) +{ + int w = VIPS_IMAGE_N_ELEMENTS( in ); + int es = VIPS_IMAGE_SIZEOF_ELEMENT( in ); + + int x, y; + PEL *p; + + p = (PEL *) in->data; + for( y = 0; y < in->Ysize; y++ ) { + for( x = 0; x < w; x++ ) { + if( x > 0 ) + fprintf( fp, "%s", sep ); + + switch( in->BandFmt ) { + case VIPS_FORMAT_UCHAR: + PRINT_INT( unsigned char ); break; + case VIPS_FORMAT_CHAR: + PRINT_INT( char ); break; + case VIPS_FORMAT_USHORT: + PRINT_INT( unsigned short ); break; + case VIPS_FORMAT_SHORT: + PRINT_INT( short ); break; + case VIPS_FORMAT_UINT: + PRINT_INT( unsigned int ); break; + case VIPS_FORMAT_INT: + PRINT_INT( int ); break; + case VIPS_FORMAT_FLOAT: + PRINT_FLOAT( float ); break; + case VIPS_FORMAT_DOUBLE: + PRINT_FLOAT( double ); break; + case VIPS_FORMAT_COMPLEX: + PRINT_COMPLEX( float ); break; + case VIPS_FORMAT_DPCOMPLEX: + PRINT_COMPLEX( double ); break; + + default: + g_assert( 0 ); + } + + p += es; + } + + fprintf( fp, "\n" ); + } + + return( 0 ); +} + +int +vips__csv_write( VipsImage *in, const char *filename, const char *separator ) +{ + FILE *fp; + + if( vips_check_mono( "vips2csv", in ) || + vips_check_uncoded( "vips2csv", in ) || + vips_image_wio_input( in ) ) + return( -1 ); + + if( !(fp = vips__file_open_write( filename, TRUE )) ) + return( -1 ); + if( vips2csv( in, fp, separator ) ) { + fclose( fp ); + return( -1 ); + } + fclose( fp ); + + return( 0 ); +} + + diff --git a/libvips/foreign/csv.h b/libvips/foreign/csv.h new file mode 100644 index 00000000..88e01cf5 --- /dev/null +++ b/libvips/foreign/csv.h @@ -0,0 +1,51 @@ +/* common defs for csv read/write + */ + +/* + + Copyright (C) 1991-2005 The National Gallery + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + */ + +/* + + These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk + + */ + +#ifndef VIPS_CSV_H +#define VIPS_CSV_H + +#ifdef __cplusplus +extern "C" { +#endif /*__cplusplus*/ + +extern const char *vips__foreign_csv_suffs[]; + +int vips__csv_read( const char *filename, VipsImage *out, + int skip, int lines, const char *whitespace, const char *separator ); +int vips__csv_read_header( const char *filename, VipsImage *out, + int skip, int lines, const char *whitespace, const char *separator ); + +int vips__csv_write( VipsImage *in, const char *filename, + const char *separator ); + +#ifdef __cplusplus +} +#endif /*__cplusplus*/ + +#endif /*VIPS_CSV_H*/ diff --git a/libvips/foreign/csvload.c b/libvips/foreign/csvload.c new file mode 100644 index 00000000..6482f1b9 --- /dev/null +++ b/libvips/foreign/csvload.c @@ -0,0 +1,227 @@ +/* load csv from a file + * + * 5/12/11 + * - from csvload.c + */ + +/* + + This file is part of VIPS. + + VIPS is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + */ + +/* + + These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk + + */ + +/* +#define DEBUG + */ + +#ifdef HAVE_CONFIG_H +#include +#endif /*HAVE_CONFIG_H*/ +#include + +#include +#include +#include + +#include +#include +#include + +#include "csv.h" + +typedef struct _VipsForeignLoadCsv { + VipsForeignLoad parent_object; + + /* Filename for load. + */ + char *filename; + + int skip; + int lines; + const char *whitespace; + const char *separator; + +} VipsForeignLoadCsv; + +typedef VipsForeignLoadClass VipsForeignLoadCsvClass; + +G_DEFINE_TYPE( VipsForeignLoadCsv, vips_foreign_load_csv, + VIPS_TYPE_FOREIGN_LOAD ); + +static VipsForeignFlags +vips_foreign_load_csv_get_flags_filename( const char *filename ) +{ + return( 0 ); +} + +static VipsForeignFlags +vips_foreign_load_csv_get_flags( VipsForeignLoad *load ) +{ + VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) load; + + return( vips_foreign_load_csv_get_flags_filename( csv->filename ) ); +} + +static int +vips_foreign_load_csv_header( VipsForeignLoad *load ) +{ + VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) load; + + if( vips__csv_read_header( csv->filename, load->out, + csv->skip, csv->lines, csv->whitespace, csv->separator ) ) + return( -1 ); + + return( 0 ); +} + +static int +vips_foreign_load_csv_load( VipsForeignLoad *load ) +{ + VipsForeignLoadCsv *csv = (VipsForeignLoadCsv *) load; + + if( vips__csv_read( csv->filename, load->real, + csv->skip, csv->lines, csv->whitespace, csv->separator ) ) + return( -1 ); + + return( 0 ); +} + +static void +vips_foreign_load_csv_class_init( VipsForeignLoadCsvClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *object_class = (VipsObjectClass *) class; + VipsForeignClass *foreign_class = (VipsForeignClass *) class; + VipsForeignLoadClass *load_class = (VipsForeignLoadClass *) class; + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + object_class->nickname = "csvload"; + object_class->description = _( "load csv from file" ); + + foreign_class->suffs = vips__foreign_csv_suffs; + + load_class->get_flags_filename = + vips_foreign_load_csv_get_flags_filename; + load_class->get_flags = vips_foreign_load_csv_get_flags; + load_class->header = vips_foreign_load_csv_header; + load_class->load = vips_foreign_load_csv_load; + + VIPS_ARG_STRING( class, "filename", 1, + _( "Filename" ), + _( "Filename to load from" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadCsv, filename ), + NULL ); + + VIPS_ARG_INT( class, "skip", 10, + _( "Skip" ), + _( "Skip this many lines at the start of the file" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadCsv, skip ), + 0, 10000000, 0 ); + + VIPS_ARG_INT( class, "lines", 11, + _( "Lines" ), + _( "Read this many lines from the file" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadCsv, lines ), + -1, 10000000, 0 ); + + VIPS_ARG_STRING( class, "whitespace", 12, + _( "Whitespace" ), + _( "Set of whitespace characters" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadCsv, whitespace ), + " " ); + + VIPS_ARG_STRING( class, "separator", 13, + _( "Separator" ), + _( "Set of separator characters" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsForeignLoadCsv, separator ), + ";,\t" ); +} + +static void +vips_foreign_load_csv_init( VipsForeignLoadCsv *csv ) +{ + csv->lines = -1; + csv->whitespace = g_strdup( " " ); + csv->separator = g_strdup( ";,\t" ); +} + +/** + * vips_csvload: + * @filename: file to load + * @out: output image + * @skip: skip this many lines at start of file + * @lines: read this many lines from file + * @whitespace: set of whitespace characters + * @separator: set of separator characters + * @...: %NULL-terminated list of optional named arguments + * + * Load a CSV (comma-separated values) file. The output image is always 1 + * band (monochrome), #VIPS_FORMAT_DOUBLE. + * + * Items in lines can be either floating point numbers in the C locale, or + * strings enclosed in double-quotes ("), or empty. + * You can use a backslash (\) within the quotes to escape special characters, + * such as quote marks. + * + * The reader is deliberately rather fussy: it will fail if there are any + * short lines, or if the file is too short. It will ignore lines that are + * too long. + * + * @skip sets the number of lines to skip at the start of the file. + * Default zero. + * + * @lines sets the number of lines to read from the file. Default -1, + * meaning read all lines to end of file. + * + * @whitespace sets the skippable whitespace characters. + * Default space. + * Whitespace characters are always run together. + * + * @separator sets the characters that separate fields. + * Default ;,tab. Separators are never run together. + * + * See also: vips_image_new_from_file(). + * + * Returns: 0 on success, -1 on error. + */ +int +vips_csvload( const char *filename, VipsImage **out, ... ) +{ + va_list ap; + int result; + + va_start( ap, out ); + result = vips_call_split( "csvload", ap, filename, out ); + va_end( ap ); + + return( result ); +} + + diff --git a/libvips/foreign/csvsave.c b/libvips/foreign/csvsave.c new file mode 100644 index 00000000..5636a4f9 --- /dev/null +++ b/libvips/foreign/csvsave.c @@ -0,0 +1,172 @@ +/* save to csv + * + * 2/12/11 + * - wrap a class around the csv writer + */ + +/* + + This file is part of VIPS. + + VIPS is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + */ + +/* + + These files are distributed with VIPS - http://www.vips.ecs.soton.ac.uk + + */ + +/* +#define DEBUG_VERBOSE +#define DEBUG + */ + +#ifdef HAVE_CONFIG_H +#include +#endif /*HAVE_CONFIG_H*/ +#include + +#include +#include +#include + +#include + +#include "csv.h" + +typedef struct _VipsForeignSaveCsv { + VipsForeignSave parent_object; + + /* Filename for save. + */ + char *filename; + + const char *separator; +} VipsForeignSaveCsv; + +typedef VipsForeignSaveClass VipsForeignSaveCsvClass; + +G_DEFINE_TYPE( VipsForeignSaveCsv, vips_foreign_save_csv, + VIPS_TYPE_FOREIGN_SAVE ); + +static int +vips_foreign_save_csv_build( VipsObject *object ) +{ + VipsForeignSave *save = (VipsForeignSave *) object; + VipsForeignSaveCsv *csv = (VipsForeignSaveCsv *) object; + + if( VIPS_OBJECT_CLASS( vips_foreign_save_csv_parent_class )-> + build( object ) ) + return( -1 ); + + if( vips__csv_write( save->ready, csv->filename, csv->separator ) ) + return( -1 ); + + return( 0 ); +} + +/* Save a bit of typing. + */ +#define UC VIPS_FORMAT_UCHAR +#define C VIPS_FORMAT_CHAR +#define US VIPS_FORMAT_USHORT +#define S VIPS_FORMAT_SHORT +#define UI VIPS_FORMAT_UINT +#define I VIPS_FORMAT_INT +#define F VIPS_FORMAT_FLOAT +#define X VIPS_FORMAT_COMPLEX +#define D VIPS_FORMAT_DOUBLE +#define DX VIPS_FORMAT_DPCOMPLEX + +static int bandfmt_csv[10] = { +/* UC C US S UI I F X D DX */ + UC, C, US, S, UI, I, F, X, D, DX +}; + +static void +vips_foreign_save_csv_class_init( VipsForeignSaveCsvClass *class ) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS( class ); + VipsObjectClass *object_class = (VipsObjectClass *) class; + VipsForeignClass *foreign_class = (VipsForeignClass *) class; + VipsForeignSaveClass *save_class = (VipsForeignSaveClass *) class; + + gobject_class->set_property = vips_object_set_property; + gobject_class->get_property = vips_object_get_property; + + object_class->nickname = "csvsave"; + object_class->description = _( "save image to csv file" ); + object_class->build = vips_foreign_save_csv_build; + + foreign_class->suffs = vips__foreign_csv_suffs; + + save_class->saveable = VIPS_SAVEABLE_MONO; + save_class->format_table = bandfmt_csv; + + VIPS_ARG_STRING( class, "filename", 1, + _( "Filename" ), + _( "Filename to save to" ), + VIPS_ARGUMENT_REQUIRED_INPUT, + G_STRUCT_OFFSET( VipsForeignSaveCsv, filename ), + NULL ); + + VIPS_ARG_STRING( class, "separator", 13, + _( "Separator" ), + _( "Separator characters" ), + VIPS_ARGUMENT_OPTIONAL_INPUT, + G_STRUCT_OFFSET( VipsForeignSaveCsv, separator ), + "\t" ); +} + +static void +vips_foreign_save_csv_init( VipsForeignSaveCsv *csv ) +{ + csv->separator = g_strdup( "\t" ); +} + +/** + * vips_csvsave: + * @in: image to save + * @filename: file to write to + * @separator: separator string + * @...: %NULL-terminated list of optional named arguments + * + * Writes the pixels in @in to the @filename as CSV (comma-separated values). + * The image is written + * one line of text per scanline. Complex numbers are written as + * "(real,imaginary)" and will need extra parsing I guess. Only the first band + * is written. + * + * @separator gives the string to use to separate numbers in the output. + * The default is "\\t" (tab). + * + * See also: vips_image_write_file(). + * + * Returns: 0 on success, -1 on error. + */ +int +vips_csvsave( VipsImage *in, const char *filename, ... ) +{ + va_list ap; + int result; + + va_start( ap, filename ); + result = vips_call_split( "csvsave", ap, filename ); + va_end( ap ); + + return( result ); +} diff --git a/libvips/foreign/foreign.c b/libvips/foreign/foreign.c index e17fd5ce..85e7b5e8 100644 --- a/libvips/foreign/foreign.c +++ b/libvips/foreign/foreign.c @@ -855,6 +855,18 @@ vips_foreign_convert_saveable( VipsForeignSave *save ) in = out; } + else if( in->Bands > 1 && + class->saveable == VIPS_SAVEABLE_MONO ) { + VipsImage *out; + + if( vips_extract_band( in, &out, 0, NULL ) ) { + g_object_unref( in ); + return( -1 ); + } + g_object_unref( in ); + + in = out; + } /* Else we have VIPS_SAVEABLE_ANY and we don't chop bands down. */ @@ -1164,6 +1176,8 @@ vips_foreign_write_options( VipsImage *in, const char *filename ) void vips_foreign_operation_init( void ) { + extern GType vips_foreign_load_csv_get_type( void ); + extern GType vips_foreign_save_csv_get_type( void ); extern GType vips_foreign_load_fits_get_type( void ); extern GType vips_foreign_save_fits_get_type( void ); extern GType vips_foreign_load_analyze_get_type( void ); @@ -1182,6 +1196,8 @@ vips_foreign_operation_init( void ) extern GType vips_foreign_save_raw_get_type( void ); extern GType vips_foreign_save_rawfd_get_type( void ); + vips_foreign_load_csv_get_type(); + vips_foreign_save_csv_get_type(); vips_foreign_load_analyze_get_type(); vips_foreign_load_raw_get_type(); vips_foreign_save_raw_get_type(); diff --git a/libvips/format/im_csv2vips.c b/libvips/format/im_csv2vips.c index 50c26e90..67e4fbcc 100644 --- a/libvips/format/im_csv2vips.c +++ b/libvips/format/im_csv2vips.c @@ -1,20 +1,7 @@ /* Read a csv file. * - * 19/12/05 JC - * - hacked from ppm reader - * 11/9/06 - * - now distingushes whitespace and separators, so we can have blank - * fields - * 20/9/06 - * - oop, unquoted trailing columns could get missed - * 17/5/07 - * - added im_csv2vips_header() - * 4/2/10 - * - gtkdoc - * 1/3/10 - * - allow lines that end with EOF - * 23/9/11 - * - allow quoted strings, including escaped quotes + * 16/12/11 + * - just a stub */ /* @@ -56,316 +43,6 @@ #include -/* Skip to the start of the next line (ie. read until we see a '\n'), return - * zero if we are at EOF. - * - * Files can end with EOF or with \nEOF. Tricky! - */ -static int -skip_line( FILE *fp ) -{ - int ch; - - /* Are we at a delayed EOF? See below. - */ - if( (ch = fgetc( fp )) == EOF ) - return( 0 ); - ungetc( ch, fp ); - - /* If we hit EOF and no \n, wait until the next call to report EOF. - */ - while( (ch = fgetc( fp )) != '\n' && ch != EOF ) - ; - - return( -1 ); -} - -static int -skip_white( FILE *fp, const char whitemap[256] ) -{ - int ch; - - do { - ch = fgetc( fp ); - } while (ch != EOF && ch != '\n' && whitemap[ch] ); - - ungetc( ch, fp ); - - return( ch ); -} - -static int -skip_to_quote( FILE *fp ) -{ - int ch; - - do { - ch = fgetc( fp ); - - /* Ignore \" in strings. - */ - if( ch == '\\' ) - ch = fgetc( fp ); - else if( ch == '"' ) - break; - } while( ch != EOF && ch != '\n' ); - - ungetc( ch, fp ); - - return( ch ); -} - -static int -skip_to_sep( FILE *fp, const char sepmap[256] ) -{ - int ch; - - do { - ch = fgetc( fp ); - } while (ch != EOF && ch != '\n' && !sepmap[ch] ); - - ungetc( ch, fp ); - - return( ch ); -} - -/* Read a single item. Syntax is: - * - * element : - * whitespace* item whitespace* [EOF|EOL|separator] - * - * item : - * double | - * "anything" | - * empty - * - * the anything in quotes can contain " escaped with \ - * - * Return the char that caused failure on fail (EOF or \n). - */ -static int -read_double( FILE *fp, const char whitemap[256], const char sepmap[256], - int lineno, int colno, double *out ) -{ - int ch; - - /* The fscanf() may change this ... but all other cases need a zero. - */ - *out = 0; - - ch = skip_white( fp, whitemap ); - if( ch == EOF || ch == '\n' ) - return( ch ); - - if( ch == '"' ) { - (void) fgetc( fp ); - ch = skip_to_quote( fp ); - ch = fgetc( fp ); - } - else if( !sepmap[ch] && - fscanf( fp, "%lf", out ) != 1 ) { - /* Only a warning, since (for example) exported spreadsheets - * will often have text or date fields. - */ - im_warn( "im_csv2vips", - _( "error parsing number, line %d, column %d" ), - lineno, colno ); - - /* Step over the bad data to the next separator. - */ - ch = skip_to_sep( fp, sepmap ); - } - - /* Don't need to check result, we have read a field successfully. - */ - ch = skip_white( fp, whitemap ); - - /* If it's a separator, we have to step over it. - */ - if( ch != EOF && sepmap[ch] ) - (void) fgetc( fp ); - - return( 0 ); -} - -static int -read_csv( FILE *fp, IMAGE *out, - int start_skip, - const char *whitespace, const char *separator, - int lines ) -{ - int i; - char whitemap[256]; - char sepmap[256]; - const char *p; - fpos_t pos; - int columns; - int ch; - double d; - double *buf; - int y; - - /* Make our char maps. - */ - for( i = 0; i < 256; i++ ) { - whitemap[i] = 0; - sepmap[i] = 0; - } - for( p = whitespace; *p; p++ ) - whitemap[(int) *p] = 1; - for( p = separator; *p; p++ ) - sepmap[(int) *p] = 1; - - /* Skip first few lines. - */ - for( i = 0; i < start_skip; i++ ) - if( !skip_line( fp ) ) { - im_error( "im_csv2vips", - "%s", _( "end of file while skipping start" ) ); - return( -1 ); - } - - /* Parse the first line to get number of columns. Only bother checking - * fgetpos() the first time we use it: assume it's working after this. - */ - if( fgetpos( fp, &pos ) ) { - im_error_system( errno, "im_csv2vips", - "%s", _( "unable to seek" ) ); - return( -1 ); - } - for( columns = 0; - (ch = read_double( fp, whitemap, sepmap, - start_skip + 1, columns + 1, &d )) == 0; - columns++ ) - ; - fsetpos( fp, &pos ); - - if( columns == 0 ) { - im_error( "im_csv2vips", "%s", _( "empty line" ) ); - return( -1 ); - } - if( ch == -2 ) - /* Failed to parse a number. - */ - return( -1 ); - - /* If lines is -1, we have to parse the whole file to get the - * number of lines out. - */ - if( lines == -1 ) { - fgetpos( fp, &pos ); - for( lines = 0; skip_line( fp ); lines++ ) - ; - fsetpos( fp, &pos ); - } - - im_initdesc( out, columns, lines, 1, - IM_BBITS_DOUBLE, IM_BANDFMT_DOUBLE, - IM_CODING_NONE, IM_TYPE_B_W, 1.0, 1.0, 0, 0 ); - - if( im_outcheck( out ) || - im_setupout( out ) || - !(buf = IM_ARRAY( out, IM_IMAGE_N_ELEMENTS( out ), double )) ) - return( -1 ); - - for( y = 0; y < lines; y++ ) { - int x; - - for( x = 0; x < columns; x++ ) { - ch = read_double( fp, whitemap, sepmap, - y + start_skip + 1, x + 1, &d ); - if( ch == EOF ) { - im_error( "im_csv2vips", - "%s", _( "unexpected end of file" ) ); - return( -1 ); - } - else if( ch == '\n' ) { - im_error( "im_csv2vips", - "%s", _( "unexpected end of line" ) ); - return( -1 ); - } - else if( ch ) - /* Parse error. - */ - return( -1 ); - - buf[x] = d; - } - - if( im_writeline( y, out, (PEL *) buf ) ) - return( -1 ); - - /* Skip over the '\n' to the next line. - */ - skip_line( fp ); - } - - return( 0 ); -} - -/** - * im_csv2vips: - * @filename: file to load - * @out: image to write to - * - * Load a CSV (comma-separated values) file. The output image is always 1 - * band (monochrome), %VIPS_FORMAT_DOUBLE. - * - * Items in lines can be either floating point numbers in the C locale, or - * strings enclosed in double-quotes ("), or empty. - * You can use a backslash (\) within the quotes to escape special characters, - * such as quote marks. - * - * The reader is deliberately rather fussy: it will fail if there are any - * short lines, or if the file is too short. It will ignore lines that are - * too long. - * - * Read options can be embedded in the filename. The options can be given - * in any order and are: - * - * - * - * - * skip:lines-to-skip The number of lines to skip at - * the start of the file. Default zero. - * - * - * - * - * line:lines-to-read - * The number of lines to read from the file. Default -1, meaning read to end of - * file. - * - * - * - * - * whi:whitespace-characters - * The skippable whitespace characters. Default space. - * Whitespace characters are always run together. - * - * - * - * - * sep:separator-characters - * The characters that separate fields. Default ;,tab. - * Separators are never run together. - * - * - * - * - * For example: - * - * |[ - * im_csv2vips( "fred.csv:skip:58,sep:\,,line:3", out ); - * ]| - * - * Will read three lines starting at line 59, with comma as the only - * allowed separator. Note that the ',' has to be escaped with a backslash. - * - * See also: #VipsFormat, im_vips2csv(), im_read_dmask(), im_ppm2vips(). - * - * Returns: 0 on success, -1 on error. - */ int im_csv2vips( const char *filename, IMAGE *out ) { @@ -379,7 +56,7 @@ im_csv2vips( const char *filename, IMAGE *out ) char name[FILENAME_MAX]; char mode[FILENAME_MAX]; char *p, *q, *r; - FILE *fp; + VipsImage *t; /* Parse mode string. */ @@ -396,33 +73,18 @@ im_csv2vips( const char *filename, IMAGE *out ) lines = atoi( r ); } - if( !(fp = im__file_open_read( name, NULL, TRUE )) ) + if( vips_csvload( filename, &t, + "skip", start_skip, + "lines", lines, + "whitespace", whitespace, + "separator", separator, + NULL ) ) return( -1 ); - if( read_csv( fp, out, start_skip, whitespace, separator, lines ) ) { - fclose( fp ); + if( vips_image_write( t, out ) ) { + g_object_unref( t ); return( -1 ); } - fclose( fp ); - - return( 0 ); -} - -/* We can't just read the header of a CSV. Instead, we read to a temp image, - * then copy just the header to the output. - */ -static int -csv2vips_header( const char *filename, IMAGE *out ) -{ - IMAGE *t; - - if( !(t = im_open( "im_csv2vips_header", "p" )) ) - return( -1 ); - if( im_csv2vips( filename, t ) || - im_cp_desc( out, t ) ) { - im_close( t ); - return( -1 ); - } - im_close( t ); + g_object_unref( t ); return( 0 ); } @@ -443,7 +105,6 @@ vips_format_csv_class_init( VipsFormatCsvClass *class ) object_class->nickname = "csv"; object_class->description = _( "CSV" ); - format_class->header = csv2vips_header; format_class->load = im_csv2vips; format_class->save = im_vips2csv; format_class->suffs = csv_suffs; diff --git a/libvips/format/im_vips2csv.c b/libvips/format/im_vips2csv.c index 0b0db0a1..3c720b7a 100644 --- a/libvips/format/im_vips2csv.c +++ b/libvips/format/im_vips2csv.c @@ -1,11 +1,7 @@ /* Write a csv file. * - * 9/6/06 - * - hacked from im_debugim - * 23/10/06 - * - allow separator to be specified (default "\t", ) - * 17/11/06 - * - oops, was broken + * 16/12/11 + * - just a stub */ /* @@ -39,101 +35,8 @@ #endif /*HAVE_CONFIG_H*/ #include -#include -#include - #include -#define PRINT_INT( TYPE ) fprintf( fp, "%d", *((TYPE*)p) ); -#define PRINT_FLOAT( TYPE ) fprintf( fp, "%g", *((TYPE*)p) ); -#define PRINT_COMPLEX( TYPE ) fprintf( fp, "(%g, %g)", \ - ((TYPE*)p)[0], ((TYPE*)p)[1] ); - -static int -vips2csv( IMAGE *in, FILE *fp, const char *sep ) -{ - int w = IM_IMAGE_N_ELEMENTS( in ); - int es = IM_IMAGE_SIZEOF_ELEMENT( in ); - - int x, y; - PEL *p; - - p = (PEL *) in->data; - for( y = 0; y < in->Ysize; y++ ) { - for( x = 0; x < w; x++ ) { - if( x > 0 ) - fprintf( fp, "%s", sep ); - - switch( in->BandFmt ) { - case IM_BANDFMT_UCHAR: - PRINT_INT( unsigned char ); break; - case IM_BANDFMT_CHAR: - PRINT_INT( char ); break; - case IM_BANDFMT_USHORT: - PRINT_INT( unsigned short ); break; - case IM_BANDFMT_SHORT: - PRINT_INT( short ); break; - case IM_BANDFMT_UINT: - PRINT_INT( unsigned int ); break; - case IM_BANDFMT_INT: - PRINT_INT( int ); break; - case IM_BANDFMT_FLOAT: - PRINT_FLOAT( float ); break; - case IM_BANDFMT_DOUBLE: - PRINT_FLOAT( double ); break; - case IM_BANDFMT_COMPLEX: - PRINT_COMPLEX( float ); break; - case IM_BANDFMT_DPCOMPLEX: - PRINT_COMPLEX( double ); break; - - default: - assert( 0 ); - } - - p += es; - } - - fprintf( fp, "\n" ); - } - - return( 0 ); -} - -/** - * im_vips2csv: - * @in: image to save - * @filename: file to write to - * - * Save a CSV (comma-separated values) file. The image is written - * one line of text per scanline. Complex numbers are written as - * "(real,imaginary)" and will need extra parsing I guess. The image must - * have a single band. - * - * Write options can be embedded in the filename. The options can be given - * in any order and are: - * - * - * - * - * sep:separator-string - * The string to use to separate numbers in the output. - * The default is "\\t" (tab). - * - * - * - * - * For example: - * - * |[ - * im_csv2vips( in, "fred.csv:sep:\t" ); - * ]| - * - * Will write to fred.csv, separating numbers with tab characters. - * - * See also: #VipsFormat, im_csv2vips(), im_write_dmask(), im_vips2ppm(). - * - * Returns: 0 on success, -1 on error. - */ int im_vips2csv( IMAGE *in, const char *filename ) { @@ -141,7 +44,6 @@ im_vips2csv( IMAGE *in, const char *filename ) char name[FILENAME_MAX]; char mode[FILENAME_MAX]; - FILE *fp; char *p, *q, *r; /* Parse mode string. @@ -153,19 +55,8 @@ im_vips2csv( IMAGE *in, const char *filename ) separator = r; } - if( im_incheck( in ) || - im_check_mono( "im_vips2csv", in ) || - im_check_uncoded( "im_vips2csv", in ) ) + if( vips_csvsave( in, name, "separator", separator, NULL ) ) return( -1 ); - if( !(fp = im__file_open_write( name, TRUE )) ) - return( -1 ); - if( vips2csv( in, fp, separator ) ) { - fclose( fp ); - return( -1 ); - } - - fclose( fp ); - return( 0 ); } diff --git a/libvips/include/vips/foreign.h b/libvips/include/vips/foreign.h index ae8e3eac..5d156caf 100644 --- a/libvips/include/vips/foreign.h +++ b/libvips/include/vips/foreign.h @@ -187,6 +187,7 @@ gboolean vips_foreign_is_a( const char *loader, const char *filename ); /** * VipsSaveable: + * @VIPS_SAVEABLE_MONO: 1 band (eg. CSV) * @VIPS_SAVEABLE_RGB: 1 or 3 bands (eg. PPM) * @VIPS_SAVEABLE_RGBA: 1, 2, 3 or 4 bands (eg. PNG) * @VIPS_SAVEABLE_RGB_CMYK: 1, 3 or 4 bands (eg. JPEG) @@ -195,6 +196,7 @@ gboolean vips_foreign_is_a( const char *loader, const char *filename ); * See also: #VipsForeignSave. */ typedef enum { + VIPS_SAVEABLE_MONO, VIPS_SAVEABLE_RGB, VIPS_SAVEABLE_RGBA, VIPS_SAVEABLE_RGB_CMYK, @@ -339,6 +341,11 @@ int vips_rawsave( VipsImage *in, const char *filename, ... ) int vips_rawsavefd( VipsImage *in, int fd, ... ) __attribute__((sentinel)); +int vips_csvload( const char *filename, VipsImage **out, ... ) + __attribute__((sentinel)); +int vips_csvsave( VipsImage *in, const char *filename, ... ) + __attribute__((sentinel)); + #ifdef __cplusplus } #endif /*__cplusplus*/ diff --git a/libvips/iofuncs/enumtypes.c b/libvips/iofuncs/enumtypes.c index afa97c78..665085c2 100644 --- a/libvips/iofuncs/enumtypes.c +++ b/libvips/iofuncs/enumtypes.c @@ -31,6 +31,7 @@ vips_saveable_get_type( void ) if( etype == 0 ) { static const GEnumValue values[] = { + {VIPS_SAVEABLE_MONO, "VIPS_SAVEABLE_MONO", "mono"}, {VIPS_SAVEABLE_RGB, "VIPS_SAVEABLE_RGB", "rgb"}, {VIPS_SAVEABLE_RGBA, "VIPS_SAVEABLE_RGBA", "rgba"}, {VIPS_SAVEABLE_RGB_CMYK, "VIPS_SAVEABLE_RGB_CMYK", "rgb-cmyk"},