rpms/coreutils
7
0
Fork 0
Code Issues Pull Requests Releases Activity
5adf0da3ed
coreutils/coreutils-i18n.patch
cvsdist 5adf0da3ed auto-import changelog data from coreutils-5.2.1-3.src.rpm 
Tue Mar 16 2004 Dan Walsh <dwalsh@redhat.com> 5.2.1-3
- If preserve fails, report as warning unless user requires preserve
Tue Mar 16 2004 Dan Walsh <dwalsh@redhat.com> 5.2.1-2
- Make mv default to preserve on context
Sat Mar 13 2004 Tim Waugh <twaugh@redhat.com> 5.2.1-1
- 5.2.1.
Fri Mar 12 2004 Tim Waugh <twaugh@redhat.com> 5.2.0-9
- Add '-Z' to 'ls --help' output (bug #118108).
Fri Mar 05 2004 Tim Waugh <twaugh@redhat.com>
- Fix deref-args test case for rebuilding under SELinux (bug #117556).
2004-09-09 03:58:39 +00:00

4098 lines
100 KiB
Diff
Raw Blame History

--- coreutils-5.1.3/lib/linebuffer.h.i18n 2003-06-18 07:58:01.000000000 +0100
+++ coreutils-5.1.3/lib/linebuffer.h 2004-02-16 15:36:40.000000000 +0000
@@ -22,6 +22,11 @@
# include <stdio.h>
+/* Get mbstate_t. */
+# if HAVE_WCHAR_H
+# include <wchar.h>
+# endif
+
/* A `struct linebuffer' holds a line of text. */
struct linebuffer
@@ -29,6 +34,9 @@
size_t size; /* Allocated. */
size_t length; /* Used. */
char *buffer;
+# if HAVE_WCHAR_H
+ mbstate_t state;
+# endif
};
/* Initialize linebuffer LINEBUFFER for use. */
--- coreutils-5.1.3/src/cut.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/cut.c 2004-02-16 15:45:41.000000000 +0000
@@ -29,6 +29,11 @@
#include <assert.h>
#include <getopt.h>
#include <sys/types.h>
+
+/* Get mbstate_t, mbrtowc(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
#include "system.h"
#include "error.h"
@@ -37,6 +42,18 @@
#include "quote.h"
#include "xstrndup.h"
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+ installation; work around this configuration error. */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# undef MB_LEN_MAX
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "cut"
@@ -67,6 +84,52 @@
} \
while (0)
+/* Refill the buffer BUF to get a multibyte character. */
+#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \
+ do \
+ { \
+ if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \
+ { \
+ memmove (BUF, BUFPOS, BUFLEN); \
+ BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \
+ BUFPOS = BUF; \
+ } \
+ } \
+ while (0)
+
+/* Get wide character on BUFPOS. BUFPOS is not included after that.
+ If byte sequence is not valid as a character, CONVFAIL is 1. Otherwise 0. */
+#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \
+ do \
+ { \
+ mbstate_t state_bak; \
+ \
+ if (BUFLEN < 1) \
+ { \
+ WC = WEOF; \
+ break; \
+ } \
+ \
+ /* Get a wide character. */ \
+ CONVFAIL = 0; \
+ state_bak = STATE; \
+ MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \
+ \
+ switch (MBLENGTH) \
+ { \
+ case (size_t)-1: \
+ case (size_t)-2: \
+ CONVFAIL++; \
+ STATE = state_bak; \
+ /* Fall througn. */ \
+ \
+ case 0: \
+ MBLENGTH = 1; \
+ break; \
+ } \
+ } \
+ while (0)
+
struct range_pair
{
size_t lo;
@@ -85,7 +148,7 @@
/* The number of bytes allocated for FIELD_1_BUFFER. */
static size_t field_1_bufsize;
-/* The largest field or byte index used as an endpoint of a closed
+/* The largest byte, character or field index used as an endpoint of a closed
or degenerate range specification; this doesn't include the starting
index of right-open-ended ranges. For example, with either range spec
`2-5,9-', `2-3,5,9-' this variable would be set to 5. */
@@ -97,10 +160,11 @@
/* This is a bit vector.
In byte mode, which bytes to output.
+ In character mode, which characters to output.
In field mode, which DELIM-separated fields to output.
- Both bytes and fields are numbered starting with 1,
+ Bytes, characters and fields are numbered starting with 1,
so the zeroth bit of this array is unused.
- A field or byte K has been selected if
+ A byte, character or field K has been selected if
(K <= MAX_RANGE_ENDPOINT and is_printable_field(K))
|| (EOL_RANGE_START > 0 && K >= EOL_RANGE_START). */
static unsigned char *printable_field;
@@ -109,9 +173,12 @@
{
undefined_mode,
- /* Output characters that are in the given bytes. */
+ /* Output bytes that are at the given positions. */
byte_mode,
+ /* Output characters that are at the given positions. */
+ character_mode,
+
/* Output the given delimeter-separated fields. */
field_mode
};
@@ -121,6 +188,13 @@
static enum operating_mode operating_mode;
+/* If nonzero, when in byte mode, don't split multibyte characters. */
+static int byte_mode_character_aware;
+
+/* If nonzero, the function for single byte locale is work
+ if this program runs on multibyte locale. */
+static int force_singlebyte_mode;
+
/* If true do not output lines containing no delimeter characters.
Otherwise, all such lines are printed. This option is valid only
with field mode. */
@@ -128,6 +202,9 @@
/* The delimeter character for field mode. */
static unsigned char delim;
+#if HAVE_WCHAR_H
+static wchar_t wcdelim;
+#endif
/* True if the --output-delimiter=STRING option was specified. */
static bool output_delimiter_specified;
@@ -199,7 +276,7 @@
-f, --fields=LIST output only these fields; also print any line\n\
that contains no delimiter character, unless\n\
the -s option is specified\n\
- -n (ignored)\n\
+ -n with -b: don't split multibyte characters\n\
"), stdout);
fputs (_("\
-s, --only-delimited do not print lines not containing delimiters\n\
@@ -327,7 +404,7 @@
in_digits = false;
/* Starting a range. */
if (dash_found)
- FATAL_ERROR (_("invalid byte or field list"));
+ FATAL_ERROR (_("invalid byte, character or field list"));
dash_found = true;
fieldstr++;
@@ -352,14 +429,16 @@
if (value == 0)
{
/* `n-'. From `initial' to end of line. */
- eol_range_start = initial;
+ if (eol_range_start == 0 ||
+ (eol_range_start != 0 && eol_range_start > initial))
+ eol_range_start = initial;
field_found = true;
}
else
{
/* `m-n' or `-n' (1-n). */
if (value < initial)
- FATAL_ERROR (_("invalid byte or field list"));
+ FATAL_ERROR (_("invalid byte, character or field list"));
/* Is there already a range going to end of line? */
if (eol_range_start != 0)
@@ -434,6 +513,9 @@
if (operating_mode == byte_mode)
error (0, 0,
_("byte offset %s is too large"), quote (bad_num));
+ else if (operating_mode == character_mode)
+ error (0, 0,
+ _("character offset %s is too large"), quote (bad_num));
else
error (0, 0,
_("field number %s is too large"), quote (bad_num));
@@ -445,7 +527,7 @@
fieldstr++;
}
else
- FATAL_ERROR (_("invalid byte or field list"));
+ FATAL_ERROR (_("invalid byte, character or field list"));
}
max_range_endpoint = 0;
@@ -515,7 +597,7 @@
print_delimiter = false;
while (1)
{
- register int c; /* Each character from the file. */
+ int c; /* Each character from the file. */
c = getc (stream);
@@ -549,6 +631,63 @@
}
}
+#if HAVE_MBRTOWC
+/* This function is in use for the following case.
+
+ 1. Read from the stream STREAM, printing to standard output any selected
+ characters.
+
+ 2. Read from stream STREAM, printing to standard output any selected bytes,
+ without splitting multibyte characters. */
+
+static void
+cut_characters_or_cut_bytes_no_split (FILE *stream)
+{
+ int idx; /* number of bytes or characters in the line so far. */
+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
+ char *bufpos; /* Next read position of BUF. */
+ size_t buflen; /* The length of the byte sequence in buf. */
+ wint_t wc; /* A gotten wide character. */
+ size_t mblength; /* The byte size of a multibyte character which shows
+ as same character as WC. */
+ mbstate_t state; /* State of the stream. */
+ int convfail; /* 1, when conversion is failed. Otherwise 0. */
+
+ idx = 0;
+ buflen = 0;
+ bufpos = buf;
+ memset (&state, '\0', sizeof(mbstate_t));
+
+ while (1)
+ {
+ REFILL_BUFFER (buf, bufpos, buflen, stream);
+
+ GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail);
+
+ if (wc == WEOF)
+ {
+ if (idx > 0)
+ putchar ('\n');
+ break;
+ }
+ else if (wc == L'\n')
+ {
+ putchar ('\n');
+ idx = 0;
+ }
+ else
+ {
+ idx += (operating_mode == byte_mode) ? mblength : 1;
+ if (print_kth (idx, NULL))
+ fwrite (bufpos, mblength, sizeof(char), stdout);
+ }
+
+ buflen -= mblength;
+ bufpos += mblength;
+ }
+}
+#endif
+
/* Read from stream STREAM, printing to standard output any selected fields. */
static void
@@ -669,13 +808,192 @@
}
}
+#if HAVE_MBRTOWC
+static void
+cut_fields_mb (FILE *stream)
+{
+ int c;
+ unsigned int field_idx;
+ int found_any_selected_field;
+ int buffer_first_field;
+ int empty_input;
+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
+ char *bufpos; /* Next read position of BUF. */
+ size_t buflen; /* The length of the byte sequence in buf. */
+ wint_t wc; /* A gotten wide character. */
+ size_t mblength; /* The byte size of a multibyte character which shows
+ as same character as WC. */
+ mbstate_t state; /* State of the stream. */
+ int convfail; /* 1, when conversion is failed. Otherwise 0. */
+
+ found_any_selected_field = 0;
+ field_idx = 1;
+ bufpos = buf;
+ buflen = 0;
+ memset (&state, '\0', sizeof(mbstate_t));
+
+ c = getc (stream);
+ empty_input = (c == EOF);
+ if (c != EOF)
+ ungetc (c, stream);
+ else
+ wc = WEOF;
+
+ /* To support the semantics of the -s flag, we may have to buffer
+ all of the first field to determine whether it is `delimited.'
+ But that is unnecessary if all non-delimited lines must be printed
+ and the first field has been selected, or if non-delimited lines
+ must be suppressed and the first field has *not* been selected.
+ That is because a non-delimited line has exactly one field. */
+ buffer_first_field = (suppress_non_delimited ^ !print_kth (1, NULL));
+
+ while (1)
+ {
+ if (field_idx == 1 && buffer_first_field)
+ {
+ int len = 0;
+
+ while (1)
+ {
+ REFILL_BUFFER (buf, bufpos, buflen, stream);
+
+ GET_NEXT_WC_FROM_BUFFER
+ (wc, bufpos, buflen, mblength, state, convfail);
+
+ if (wc == WEOF)
+ break;
+
+ field_1_buffer = xrealloc (field_1_buffer, len + mblength);
+ memcpy (field_1_buffer + len, bufpos, mblength);
+ len += mblength;
+ buflen -= mblength;
+ bufpos += mblength;
+
+ if (!convfail && (wc == L'\n' || wc == wcdelim))
+ break;
+ }
+
+ if (wc == WEOF)
+ break;
+
+ /* If the first field extends to the end of line (it is not
+ delimited) and we are printing all non-delimited lines,
+ print this one. */
+ if (convfail || (!convfail && wc != wcdelim))
+ {
+ if (suppress_non_delimited)
+ {
+ /* Empty. */
+ }
+ else
+ {
+ fwrite (field_1_buffer, sizeof (char), len, stdout);
+ /* Make sure the output line is newline terminated. */
+ if (convfail || (!convfail && wc != L'\n'))
+ putchar ('\n');
+ }
+ continue;
+ }
+
+ if (print_kth (1, NULL))
+ {
+ /* Print the field, but not the trailing delimiter. */
+ fwrite (field_1_buffer, sizeof (char), len - 1, stdout);
+ found_any_selected_field = 1;
+ }
+ ++field_idx;
+ }
+
+ if (wc != WEOF)
+ {
+ if (print_kth (field_idx, NULL))
+ {
+ if (found_any_selected_field)
+ {
+ fwrite (output_delimiter_string, sizeof (char),
+ output_delimiter_length, stdout);
+ }
+ found_any_selected_field = 1;
+ }
+
+ while (1)
+ {
+ REFILL_BUFFER (buf, bufpos, buflen, stream);
+
+ GET_NEXT_WC_FROM_BUFFER
+ (wc, bufpos, buflen, mblength, state, convfail);
+
+ if (wc == WEOF)
+ break;
+ else if (!convfail && (wc == wcdelim || wc == L'\n'))
+ {
+ buflen -= mblength;
+ bufpos += mblength;
+ break;
+ }
+
+ if (print_kth (field_idx, NULL))
+ fwrite (bufpos, mblength, sizeof(char), stdout);
+
+ buflen -= mblength;
+ bufpos += mblength;
+ }
+ }
+
+ if ((!convfail || wc == L'\n') && buflen < 1)
+ wc = WEOF;
+
+ if (!convfail && wc == wcdelim)
+ ++field_idx;
+ else if (wc == WEOF || (!convfail && wc == L'\n'))
+ {
+ if (found_any_selected_field
+ || (!empty_input && !(suppress_non_delimited && field_idx == 1)))
+ putchar ('\n');
+ if (wc == WEOF)
+ break;
+ field_idx = 1;
+ found_any_selected_field = 0;
+ }
+ }
+}
+#endif
+
static void
cut_stream (FILE *stream)
{
- if (operating_mode == byte_mode)
- cut_bytes (stream);
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
+ {
+ switch (operating_mode)
+ {
+ case byte_mode:
+ if (byte_mode_character_aware)
+ cut_characters_or_cut_bytes_no_split (stream);
+ else
+ cut_bytes (stream);
+ break;
+
+ case character_mode:
+ cut_characters_or_cut_bytes_no_split (stream);
+ break;
+
+ case field_mode:
+ cut_fields_mb (stream);
+ break;
+
+ default:
+ abort ();
+ }
+ }
else
- cut_fields (stream);
+#endif
+ {
+ if (operating_mode == field_mode)
+ cut_fields (stream);
+ else
+ cut_bytes (stream);
+ }
}
/* Process file FILE to standard output.
@@ -724,6 +1042,8 @@
int optc, exit_status = 0;
bool delim_specified = false;
char *spec_list_string IF_LINT(= NULL);
+ char mbdelim[MB_LEN_MAX + 1];
+ size_t delimlen = 0;
initialize_main (&argc, &argv);
program_name = argv[0];
@@ -749,7 +1069,6 @@
break;
case 'b':
- case 'c':
/* Build the byte list. */
if (operating_mode != undefined_mode)
FATAL_ERROR (_("only one type of list may be specified"));
@@ -757,6 +1076,14 @@
spec_list_string = optarg;
break;
+ case 'c':
+ /* Build the character list. */
+ if (operating_mode != undefined_mode)
+ FATAL_ERROR (_("only one type of list may be specified"));
+ operating_mode = character_mode;
+ spec_list_string = optarg;
+ break;
+
case 'f':
/* Build the field list. */
if (operating_mode != undefined_mode)
@@ -768,10 +1095,35 @@
case 'd':
/* New delimiter. */
/* Interpret -d '' to mean `use the NUL byte as the delimiter.' */
- if (optarg[0] != '\0' && optarg[1] != '\0')
- FATAL_ERROR (_("the delimiter must be a single character"));
- delim = optarg[0];
- delim_specified = true;
+#if HAVE_MBRTOWC
+ {
+ if(MB_CUR_MAX > 1)
+ {
+ mbstate_t state;
+
+ memset (&state, '\0', sizeof(mbstate_t));
+ delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state);
+
+ if (delimlen == (size_t)-1 || delimlen == (size_t)-2)
+ ++force_singlebyte_mode;
+ else
+ {
+ delimlen = (delimlen < 1) ? 1 : delimlen;
+ if (wcdelim != L'\0' && *(optarg + delimlen) != '\0')
+ FATAL_ERROR (_("the delimiter must be a single character"));
+ memcpy (mbdelim, optarg, delimlen);
+ }
+ }
+
+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
+#endif
+ {
+ if (optarg[0] != '\0' && optarg[1] != '\0')
+ FATAL_ERROR (_("the delimiter must be a single character"));
+ delim = (unsigned char) optarg[0];
+ }
+ delim_specified = true;
+ }
break;
case OUTPUT_DELIMITER_OPTION:
@@ -784,6 +1136,7 @@
break;
case 'n':
+ byte_mode_character_aware = 1;
break;
case 's':
@@ -802,7 +1155,7 @@
if (operating_mode == undefined_mode)
FATAL_ERROR (_("you must specify a list of bytes, characters, or fields"));
- if (delim != '\0' && operating_mode != field_mode)
+ if (delim_specified && operating_mode != field_mode)
FATAL_ERROR (_("an input delimiter may be specified only\
when operating on fields"));
@@ -829,15 +1182,34 @@
}
if (!delim_specified)
- delim = '\t';
+ {
+ delim = '\t';
+#ifdef HAVE_MBRTOWC
+ wcdelim = L'\t';
+ mbdelim[0] = '\t';
+ mbdelim[1] = '\0';
+ delimlen = 1;
+#endif
+ }
if (output_delimiter_string == NULL)
{
- static char dummy[2];
- dummy[0] = delim;
- dummy[1] = '\0';
- output_delimiter_string = dummy;
- output_delimiter_length = 1;
+#ifdef HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
+ {
+ output_delimiter_string = xstrdup(mbdelim);
+ output_delimiter_length = delimlen;
+ }
+
+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
+#endif
+ {
+ static char dummy[2];
+ dummy[0] = delim;
+ dummy[1] = '\0';
+ output_delimiter_string = dummy;
+ output_delimiter_length = 1;
+ }
}
if (optind == argc)
--- coreutils-5.1.3/src/pr.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/pr.c 2004-02-16 15:36:40.000000000 +0000
@@ -314,12 +314,50 @@
#include <stdio.h>
#include <getopt.h>
#include <sys/types.h>
+
+/* Get MB_LEN_MAX. */
+#include <limits.h>
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+ installation; work around this configuration error. */
+#if !defined MB_LEN_MAX || MB_LEN_MAX == 1
+# define MB_LEN_MAX 16
+#endif
+
+/* Get MB_CUR_MAX. */
+#include <stdlib.h>
+
+/* Solaris 2.5 has a bug: <wchar.h> must be included before <wctype.h>. */
+/* Get mbstate_t, mbrtowc(), wcwidth(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
+/* Get iswprint(). -- for wcwidth(). */
+#if HAVE_WCTYPE_H
+# include <wctype.h>
+#endif
+#if !defined iswprint && !HAVE_ISWPRINT
+# define iswprint(wc) 1
+#endif
+
#include "system.h"
#include "error.h"
#include "mbswidth.h"
#include "posixver.h"
#include "xstrtol.h"
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
+#ifndef HAVE_DECL_WCWIDTH
+"this configure-time declaration test was not run"
+#endif
+#if !HAVE_DECL_WCWIDTH
+extern int wcwidth ();
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "pr"
@@ -412,7 +450,20 @@
#define NULLCOL (COLUMN *)0
-static int char_to_clump (int c);
+/* Funtion pointers to switch functions for single byte locale or for
+ multibyte locale. If multibyte functions do not exist in your sysytem,
+ these pointers always point the function for single byte locale. */
+static void (*print_char) (int c);
+static int (*char_to_clump) (int c);
+
+/* Functions for single byte locale. */
+static void print_char_single (int c);
+static int char_to_clump_single (int c);
+
+/* Functions for multibyte locale. */
+static void print_char_multi (int c);
+static int char_to_clump_multi (int c);
+
static int read_line (COLUMN *p);
static int print_page (void);
static int print_stored (COLUMN *p);
@@ -422,6 +473,7 @@
static void pad_across_to (int position);
static void add_line_number (COLUMN *p);
static void getoptarg (char *arg, char switch_char, char *character,
+ int *character_length, int *character_width,
int *number);
void usage (int status);
static void print_files (int number_of_files, char **av);
@@ -436,7 +488,6 @@
static void pad_down (int lines);
static void read_rest_of_line (COLUMN *p);
static void skip_read (COLUMN *p, int column_number);
-static void print_char (int c);
static void cleanup (void);
static void first_last_page (char *pages);
static void print_sep_string (void);
@@ -452,7 +503,7 @@
we store the leftmost columns contiguously in buff.
To print a line from buff, get the index of the first character
from line_vector[i], and print up to line_vector[i + 1]. */
-static char *buff;
+static unsigned char *buff;
/* Index of the position in buff where the next character
will be stored. */
@@ -556,7 +607,7 @@
static int untabify_input = FALSE;
/* (-e) The input tab character. */
-static char input_tab_char = '\t';
+static char input_tab_char[MB_LEN_MAX] = "\t";
/* (-e) Tabstops are at chars_per_tab, 2*chars_per_tab, 3*chars_per_tab, ...
where the leftmost column is 1. */
@@ -566,7 +617,10 @@
static int tabify_output = FALSE;
/* (-i) The output tab character. */
-static char output_tab_char = '\t';
+static char output_tab_char[MB_LEN_MAX] = "\t";
+
+/* (-i) The byte length of output tab character. */
+static int output_tab_char_length = 1;
/* (-i) The width of the output tab. */
static int chars_per_output_tab = 8;
@@ -640,7 +694,13 @@
static int numbered_lines = FALSE;
/* (-n) Character which follows each line number. */
-static char number_separator = '\t';
+static char number_separator[MB_LEN_MAX] = "\t";
+
+/* (-n) The byte length of the character which follows each line number. */
+static int number_separator_length = 1;
+
+/* (-n) The character width of the character which follows each line number. */
+static int number_separator_width = 0;
/* (-n) line counting starts with 1st line of input file (not with 1st
line of 1st page printed). */
@@ -693,6 +753,7 @@
-a|COLUMN|-m is a `space' and with the -J option a `tab'. */
static char *col_sep_string = "";
static int col_sep_length = 0;
+static int col_sep_width = 0;
static char *column_separator = " ";
static char *line_separator = "\t";
@@ -842,6 +903,13 @@
col_sep_length = (int) strlen (optarg_S);
col_sep_string = xmalloc (col_sep_length + 1);
strcpy (col_sep_string, optarg_S);
+
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ col_sep_width = mbswidth (col_sep_string, 0);
+ else
+#endif
+ col_sep_width = col_sep_length;
}
int
@@ -866,6 +934,21 @@
atexit (close_stdout);
+/* Define which functions are used, the ones for single byte locale or the ones
+ for multibyte locale. */
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ print_char = print_char_multi;
+ char_to_clump = char_to_clump_multi;
+ }
+ else
+#endif
+ {
+ print_char = print_char_single;
+ char_to_clump = char_to_clump_single;
+ }
+
n_files = 0;
file_names = (argc > 1
? xmalloc ((argc - 1) * sizeof (char *))
@@ -943,8 +1026,12 @@
break;
case 'e':
if (optarg)
- getoptarg (optarg, 'e', &input_tab_char,
- &chars_per_input_tab);
+ {
+ int dummy_length, dummy_width;
+
+ getoptarg (optarg, 'e', input_tab_char, &dummy_length,
+ &dummy_width, &chars_per_input_tab);
+ }
/* Could check tab width > 0. */
untabify_input = TRUE;
break;
@@ -957,8 +1044,12 @@
break;
case 'i':
if (optarg)
- getoptarg (optarg, 'i', &output_tab_char,
- &chars_per_output_tab);
+ {
+ int dummy_width;
+
+ getoptarg (optarg, 'i', output_tab_char, &output_tab_char_length,
+ &dummy_width, &chars_per_output_tab);
+ }
/* Could check tab width > 0. */
tabify_output = TRUE;
break;
@@ -985,8 +1076,8 @@
case 'n':
numbered_lines = TRUE;
if (optarg)
- getoptarg (optarg, 'n', &number_separator,
- &chars_per_number);
+ getoptarg (optarg, 'n', number_separator, &number_separator_length,
+ &number_separator_width, &chars_per_number);
break;
case 'N':
skip_count = FALSE;
@@ -1025,7 +1116,7 @@
old_s = FALSE;
/* Reset an additional input of -s, -S dominates -s */
col_sep_string = "";
- col_sep_length = 0;
+ col_sep_length = col_sep_width = 0;
use_col_separator = TRUE;
if (optarg)
separator_string (optarg);
@@ -1173,10 +1264,45 @@
a number. */
static void
-getoptarg (char *arg, char switch_char, char *character, int *number)
+getoptarg (char *arg, char switch_char, char *character, int *character_length,
+ int *character_width, int *number)
{
if (!ISDIGIT (*arg))
- *character = *arg++;
+ {
+#ifdef HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1) /* for multibyte locale. */
+ {
+ wchar_t wc;
+ size_t mblength;
+ int width;
+ mbstate_t state = {'\0'};
+
+ mblength = mbrtowc (&wc, arg, strnlen(arg, MB_LEN_MAX), &state);
+
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ *character_length = 1;
+ *character_width = 1;
+ }
+ else
+ {
+ *character_length = (mblength < 1) ? 1 : mblength;
+ width = wcwidth (wc);
+ *character_width = (width < 0) ? 0 : width;
+ }
+
+ strncpy (character, arg, *character_length);
+ arg += *character_length;
+ }
+ else /* for single byte locale. */
+#endif
+ {
+ *character = *arg++;
+ *character_length = 1;
+ *character_width = 1;
+ }
+ }
+
if (*arg)
{
long int tmp_long;
@@ -1241,7 +1367,7 @@
else
col_sep_string = column_separator;
- col_sep_length = 1;
+ col_sep_length = col_sep_width = 1;
use_col_separator = TRUE;
}
/* It's rather pointless to define a TAB separator with column
@@ -1273,11 +1399,11 @@
TAB_WIDTH (chars_per_input_tab, chars_per_number); */
/* Estimate chars_per_text without any margin and keep it constant. */
- if (number_separator == '\t')
+ if (number_separator[0] == '\t')
number_width = chars_per_number +
TAB_WIDTH (chars_per_default_tab, chars_per_number);
else
- number_width = chars_per_number + 1;
+ number_width = chars_per_number + number_separator_width;
/* The number is part of the column width unless we are
printing files in parallel. */
@@ -1292,7 +1418,7 @@
}
chars_per_column = (chars_per_line - chars_used_by_number -
- (columns - 1) * col_sep_length) / columns;
+ (columns - 1) * col_sep_width) / columns;
if (chars_per_column < 1)
error (EXIT_FAILURE, 0, _("page width too narrow"));
@@ -1421,7 +1547,7 @@
/* Enlarge p->start_position of first column to use the same form of
padding_not_printed with all columns. */
- h = h + col_sep_length;
+ h = h + col_sep_width;
/* This loop takes care of all but the rightmost column. */
@@ -1455,7 +1581,7 @@
}
else
{
- h = h_next + col_sep_length;
+ h = h_next + col_sep_width;
h_next = h + chars_per_column;
}
}
@@ -1739,9 +1865,9 @@
align_column (COLUMN *p)
{
padding_not_printed = p->start_position;
- if (padding_not_printed - col_sep_length > 0)
+ if (padding_not_printed - col_sep_width > 0)
{
- pad_across_to (padding_not_printed - col_sep_length);
+ pad_across_to (padding_not_printed - col_sep_width);
padding_not_printed = ANYWHERE;
}
@@ -2015,13 +2141,13 @@
/* May be too generous. */
buff = x2nrealloc (buff, &buff_allocated, sizeof *buff);
}
- buff[buff_current++] = (char) c;
+ buff[buff_current++] = (unsigned char) c;
}
static void
add_line_number (COLUMN *p)
{
- int i;
+ int i, j;
char *s;
int left_cut;
@@ -2044,22 +2170,24 @@
/* Tabification is assumed for multiple columns, also for n-separators,
but `default n-separator = TAB' hasn't been given priority over
equal column_width also specified by POSIX. */
- if (number_separator == '\t')
+ if (number_separator[0] == '\t')
{
i = number_width - chars_per_number;
while (i-- > 0)
(p->char_func) ((int) ' ');
}
else
- (p->char_func) ((int) number_separator);
+ for (j = 0; j < number_separator_length; j++)
+ (p->char_func) ((int) number_separator[j]);
}
else
/* To comply with POSIX, we avoid any expansion of default TAB
separator with a single column output. No column_width requirement
has to be considered. */
{
- (p->char_func) ((int) number_separator);
- if (number_separator == '\t')
+ for (j = 0; j < number_separator_length; j++)
+ (p->char_func) ((int) number_separator[j]);
+ if (number_separator[0] == '\t')
output_position = POS_AFTER_TAB (chars_per_output_tab,
output_position);
}
@@ -2219,7 +2347,7 @@
while (goal - h_old > 1
&& (h_new = POS_AFTER_TAB (chars_per_output_tab, h_old)) <= goal)
{
- putchar (output_tab_char);
+ fwrite (output_tab_char, sizeof(char), output_tab_char_length, stdout);
h_old = h_new;
}
while (++h_old <= goal)
@@ -2239,6 +2367,7 @@
{
char *s;
int l = col_sep_length;
+ int not_space_flag;
s = col_sep_string;
@@ -2252,6 +2381,7 @@
{
for (; separators_not_printed > 0; --separators_not_printed)
{
+ not_space_flag = 0;
while (l-- > 0)
{
/* 3 types of sep_strings: spaces only, spaces and chars,
@@ -2265,12 +2395,15 @@
}
else
{
+ not_space_flag = 1;
if (spaces_not_printed > 0)
print_white_space ();
putchar (*s++);
- ++output_position;
}
}
+ if (not_space_flag)
+ output_position += col_sep_width;
+
/* sep_string ends with some spaces */
if (spaces_not_printed > 0)
print_white_space ();
@@ -2297,8 +2430,9 @@
a nonspace is encountered, call print_white_space() to print the
required number of tabs and spaces. */
+
static void
-print_char (int c)
+print_char_single (int c)
{
if (tabify_output)
{
@@ -2322,6 +2456,74 @@
putchar (c);
}
+#ifdef HAVE_MBRTOWC
+static void
+print_char_multi (int c)
+{
+ static size_t mbc_pos = 0;
+ static unsigned char mbc[MB_LEN_MAX] = {'\0'};
+ static mbstate_t state = {'\0'};
+ mbstate_t state_bak;
+ wchar_t wc;
+ size_t mblength;
+ int width;
+
+ if (tabify_output)
+ {
+ state_bak = state;
+ mbc[mbc_pos++] = (unsigned char)c;
+ mblength = mbrtowc (&wc, mbc, mbc_pos, &state);
+
+ while (mbc_pos > 0)
+ {
+ switch (mblength)
+ {
+ case (size_t)-2:
+ state = state_bak;
+ return;
+
+ case (size_t)-1:
+ state = state_bak;
+ ++output_position;
+ putchar (mbc[0]);
+ memmove (mbc, mbc + 1, MB_CUR_MAX - 1);
+ --mbc_pos;
+ break;
+
+ case 0:
+ mblength = 1;
+
+ default:
+ if (wc == L' ')
+ {
+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
+ --mbc_pos;
+ ++spaces_not_printed;
+ return;
+ }
+ else if (spaces_not_printed > 0)
+ print_white_space ();
+
+ /* Nonprintables are assumed to have width 0, except L'\b'. */
+ if ((width = wcwidth (wc)) < 1)
+ {
+ if (wc == L'\b')
+ --output_position;
+ }
+ else
+ output_position += width;
+
+ fwrite (mbc, sizeof(char), mblength, stdout);
+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
+ mbc_pos -= mblength;
+ }
+ }
+ return;
+ }
+ putchar (c);
+}
+#endif
+
/* Skip to page PAGE before printing.
PAGE may be larger than total number of pages. */
@@ -2496,9 +2698,9 @@
align_empty_cols = FALSE;
}
- if (padding_not_printed - col_sep_length > 0)
+ if (padding_not_printed - col_sep_width > 0)
{
- pad_across_to (padding_not_printed - col_sep_length);
+ pad_across_to (padding_not_printed - col_sep_width);
padding_not_printed = ANYWHERE;
}
@@ -2599,9 +2801,9 @@
}
}
- if (padding_not_printed - col_sep_length > 0)
+ if (padding_not_printed - col_sep_width > 0)
{
- pad_across_to (padding_not_printed - col_sep_length);
+ pad_across_to (padding_not_printed - col_sep_width);
padding_not_printed = ANYWHERE;
}
@@ -2614,8 +2816,8 @@
if (spaces_not_printed == 0)
{
output_position = p->start_position + end_vector[line];
- if (p->start_position - col_sep_length == chars_per_margin)
- output_position -= col_sep_length;
+ if (p->start_position - col_sep_width == chars_per_margin)
+ output_position -= col_sep_width;
}
return TRUE;
@@ -2633,8 +2835,9 @@
characters in clump_buff. (e.g, the width of '\b' is -1, while the
number of characters is 1.) */
+
static int
-char_to_clump (int c)
+char_to_clump_single (int c)
{
register int *s = clump_buff;
register int i;
@@ -2643,10 +2846,10 @@
int chars;
int chars_per_c = 8;
- if (c == input_tab_char)
+ if (c == input_tab_char[0])
chars_per_c = chars_per_input_tab;
- if (c == input_tab_char || c == '\t')
+ if (c == input_tab_char[0] || c == '\t')
{
width = TAB_WIDTH (chars_per_c, input_position);
@@ -2717,6 +2920,154 @@
return chars;
}
+#ifdef HAVE_MBRTOWC
+static int
+char_to_clump_multi (int c)
+{
+ static size_t mbc_pos = 0;
+ static unsigned char mbc[MB_LEN_MAX] = {'\0'};
+ static mbstate_t state = {'\0'};
+ mbstate_t state_bak;
+ wchar_t wc;
+ size_t mblength;
+ int wc_width;
+ register int *s = clump_buff;
+ register int i, j;
+ char esc_buff[4];
+ int width;
+ int chars;
+ int chars_per_c = 8;
+
+ state_bak = state;
+ mbc[mbc_pos++] = (unsigned char)c;
+ mblength = mbrtowc (&wc, mbc, mbc_pos, &state);
+
+ width = 0;
+ chars = 0;
+ while (mbc_pos > 0)
+ {
+ switch (mblength)
+ {
+ case (size_t)-2:
+ state = state_bak;
+ return 0;
+
+ case (size_t)-1:
+ state = state_bak;
+ mblength = 1;
+
+ if (use_esc_sequence || use_cntrl_prefix)
+ {
+ width = +4;
+ chars = +4;
+ *s++ = '\\';
+ sprintf (esc_buff, "%03o", mbc[0]);
+ for (i = 0; i <= 2; ++i)
+ *s++ = (int) esc_buff[i];
+ }
+ else
+ {
+ width += 1;
+ chars += 1;
+ *s++ = mbc[0];
+ }
+ break;
+
+ case 0:
+ mblength = 1;
+ /* Fall through */
+
+ default:
+ if (memcmp (mbc, input_tab_char, mblength) == 0)
+ chars_per_c = chars_per_input_tab;
+
+ if (memcmp (mbc, input_tab_char, mblength) == 0 || c == '\t')
+ {
+ int width_inc;
+
+ width_inc = TAB_WIDTH (chars_per_c, input_position);
+ width += width_inc;
+
+ if (untabify_input)
+ {
+ for (i = width_inc; i; --i)
+ *s++ = ' ';
+ chars += width_inc;
+ }
+ else
+ {
+ for (i = 0; i < mblength; i++)
+ *s++ = mbc[i];
+ chars += mblength;
+ }
+ }
+ else if ((wc_width = wcwidth (wc)) < 1)
+ {
+ if (use_esc_sequence)
+ {
+ for (i = 0; i < mblength; i++)
+ {
+ width += 4;
+ chars += 4;
+ *s++ = '\\';
+ sprintf (esc_buff, "%03o", c);
+ for (j = 0; j <= 2; ++j)
+ *s++ = (int) esc_buff[j];
+ }
+ }
+ else if (use_cntrl_prefix)
+ {
+ if (wc < 0200)
+ {
+ width += 2;
+ chars += 2;
+ *s++ = '^';
+ *s++ = wc ^ 0100;
+ }
+ else
+ {
+ for (i = 0; i < mblength; i++)
+ {
+ width += 4;
+ chars += 4;
+ *s++ = '\\';
+ sprintf (esc_buff, "%03o", c);
+ for (j = 0; j <= 2; ++j)
+ *s++ = (int) esc_buff[j];
+ }
+ }
+ }
+ else if (wc == L'\b')
+ {
+ width += -1;
+ chars += 1;
+ *s++ = c;
+ }
+ else
+ {
+ width += 0;
+ chars += mblength;
+ for (i = 0; i < mblength; i++)
+ *s++ = mbc[i];
+ }
+ }
+ else
+ {
+ width += wc_width;
+ chars += mblength;
+ for (i = 0; i < mblength; i++)
+ *s++ = mbc[i];
+ }
+ }
+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
+ mbc_pos -= mblength;
+ }
+
+ input_position += width;
+ return chars;
+}
+#endif
+
/* We've just printed some files and need to clean up things before
looking for more options and printing the next batch of files.
--- coreutils-5.1.3/src/sort.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/sort.c 2004-02-16 15:36:40.000000000 +0000
@@ -23,10 +23,31 @@
#include <config.h>
+#include <assert.h>
#include <getopt.h>
#include <sys/types.h>
#include <signal.h>
#include <stdio.h>
+
+/* Solaris 2.5 has a bug: <wchar.h> must be included before <wctype.h>. */
+/* Get mbstate_t, mbrtowc(), wcwidth(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
+/* Get isw* functions. */
+#if HAVE_WCTYPE_H
+# include <wctype.h>
+#endif
+
+/* Get nl_langinfo(). */
+#if HAVE_LANGINFO_CODESET
+# include <langinfo.h>
+#endif
+
+/* Include this after wctype.h so that we `#undef' ISPRINT
+ (from Solaris's euc.h, from widec.h, from wctype.h) before
+ redefining and using it. */
#include "system.h"
#include "error.h"
#include "hard-locale.h"
@@ -46,6 +67,17 @@
# define getrlimit(Resource, Rlp) (-1)
#endif
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+ installation; work around this configuration error. */
+#if !defined MB_LEN_MAX || MB_LEN_MAX == 1
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "sort"
@@ -91,6 +123,7 @@
static char decimal_point;
static int th_sep; /* if CHAR_MAX + 1, then there is no thousands separator */
+static int force_general_numcompare = 0;
/* Nonzero if the corresponding locales are hard. */
static bool hard_LC_COLLATE;
@@ -109,6 +142,28 @@
#define NONZERO(x) (x != 0)
+/* get a multibyte character's byte length. */
+#define GET_BYTELEN_OF_CHAR(LIM, PTR, MBLENGTH, STATE) \
+ do \
+ { \
+ wchar_t wc; \
+ mbstate_t state_bak; \
+ \
+ state_bak = STATE; \
+ mblength = mbrtowc (&wc, PTR, LIM - PTR, &STATE); \
+ \
+ switch (MBLENGTH) \
+ { \
+ case (size_t)-1: \
+ case (size_t)-2: \
+ STATE = state_bak; \
+ /* Fall through. */ \
+ case 0: \
+ MBLENGTH = 1; \
+ } \
+ } \
+ while (0)
+
/* The kind of blanks for '-b' to skip in various options. */
enum blanktype { bl_start, bl_end, bl_both };
@@ -251,7 +306,8 @@
/* Tab character separating fields. If TAB_DEFAULT, then fields are
separated by the empty string between a non-blank character and a blank
character. */
-static int tab = TAB_DEFAULT;
+static int tab[MB_LEN_MAX + 1] = { TAB_DEFAULT };
+static size_t tab_length = 1;
/* Flag to remove consecutive duplicate lines from the output.
Only the last of a sequence of equal lines will be output. */
@@ -384,6 +440,46 @@
};
static struct tempnode *volatile temphead;
+/* Fucntion pointers. */
+static void
+(*inittables) (void);
+
+static char *
+(* begfield) (const struct line *line, const struct keyfield *key);
+
+static char *
+(* limfield) (const struct line *line, const struct keyfield *key);
+
+static int
+(*getmonth) (const char *s, size_t len);
+
+static int
+(* keycompare) (const struct line *a, const struct line *b);
+
+/* Test for white space multibyte character.
+ Set LENGTH the byte length of investigated multibyte character. */
+#if HAVE_MBRTOWC
+static int
+ismbblank (const char *str, size_t len, size_t *length)
+{
+ size_t mblength;
+ wchar_t wc;
+ mbstate_t state;
+
+ memset (&state, '\0', sizeof(mbstate_t));
+ mblength = mbrtowc (&wc, str, len, &state);
+
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ *length = 1;
+ return 0;
+ }
+
+ *length = (mblength < 1) ? 1 : mblength;
+ return iswblank (wc);
+}
+#endif
+
/* Clean up any remaining temporary files. */
static void
@@ -521,7 +617,7 @@
}
}
-#if HAVE_NL_LANGINFO
+#if HAVE_LANGINFO_CODESET
static int
struct_month_cmp (const void *m1, const void *m2)
@@ -536,7 +632,7 @@
/* Initialize the character class tables. */
static void
-inittables (void)
+inittables_uni (void)
{
int i;
@@ -574,6 +670,64 @@
#endif
}
+#if HAVE_MBRTOWC
+static void
+inittables_mb (void)
+{
+ int i, j, k, l;
+ char *name, *s;
+ size_t s_len, mblength;
+ char mbc[MB_LEN_MAX];
+ wchar_t wc, pwc;
+ mbstate_t state_mb, state_wc;
+
+ for (i = 0; i < MONTHS_PER_YEAR; i++)
+ {
+ s = (char *) nl_langinfo (ABMON_1 + i);
+ s_len = strlen (s);
+ monthtab[i].name = name = (char *) xmalloc (s_len + 1);
+ monthtab[i].val = i + 1;
+
+ memset (&state_mb, '\0', sizeof (mbstate_t));
+ memset (&state_wc, '\0', sizeof (mbstate_t));
+
+ for (j = 0; j < s_len;)
+ {
+ if (!ismbblank (s + j, s_len - j, &mblength))
+ break;
+ j += mblength;
+ }
+
+ for (k = 0; j < s_len;)
+ {
+ mblength = mbrtowc (&wc, (s + j), (s_len - j), &state_mb);
+ assert (mblength != (size_t)-1 && mblength != (size_t)-2);
+ if (mblength == 0)
+ break;
+
+ pwc = towupper (wc);
+ if (pwc == wc)
+ {
+ memcpy (mbc, s + j, mblength);
+ j += mblength;
+ }
+ else
+ {
+ j += mblength;
+ mblength = wcrtomb (mbc, pwc, &state_wc);
+ assert (mblength != (size_t)0 && mblength != (size_t)-1);
+ }
+
+ for (l = 0; l < mblength; l++)
+ name[k++] = mbc[l];
+ }
+ name[k] = '\0';
+ }
+ qsort ((void *) monthtab, MONTHS_PER_YEAR,
+ sizeof (struct month), struct_month_cmp);
+}
+#endif
+
/* Specify the amount of main memory to use when sorting. */
static void
specify_sort_size (char const *s)
@@ -784,7 +938,7 @@
by KEY in LINE. */
static char *
-begfield (const struct line *line, const struct keyfield *key)
+begfield_uni (const struct line *line, const struct keyfield *key)
{
register char *ptr = line->text, *lim = ptr + line->length - 1;
register size_t sword = key->sword;
@@ -794,10 +948,10 @@
/* The leading field separator itself is included in a field when -t
is absent. */
- if (tab != TAB_DEFAULT)
+ if (tab[0] != TAB_DEFAULT)
while (ptr < lim && sword--)
{
- while (ptr < lim && *ptr != tab)
+ while (ptr < lim && *ptr != tab[0])
++ptr;
if (ptr < lim)
++ptr;
@@ -825,11 +979,70 @@
return ptr;
}
+#if HAVE_MBRTOWC
+static char *
+begfield_mb (const struct line *line, const struct keyfield *key)
+{
+ int i;
+ char *ptr = line->text, *lim = ptr + line->length - 1;
+ size_t sword = key->sword;
+ size_t schar = key->schar;
+ size_t mblength;
+ mbstate_t state;
+
+ memset (&state, '\0', sizeof(mbstate_t));
+
+ if (tab[0] != TAB_DEFAULT)
+ while (ptr < lim && sword--)
+ {
+ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ ptr += mblength;
+ }
+ if (ptr < lim)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ ptr += mblength;
+ }
+ }
+ else
+ while (ptr < lim && sword--)
+ {
+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
+ ptr += mblength;
+ if (ptr < lim)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ ptr += mblength;
+ }
+ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength))
+ ptr += mblength;
+ }
+
+ if (key->skipsblanks)
+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
+ ptr += mblength;
+
+ for (i = 0; i < schar; i++)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+
+ if (ptr + mblength > lim)
+ break;
+ else
+ ptr += mblength;
+ }
+
+ return ptr;
+}
+#endif
+
/* Return the limit of (a pointer to the first character after) the field
in LINE specified by KEY. */
static char *
-limfield (const struct line *line, const struct keyfield *key)
+limfield_uni (const struct line *line, const struct keyfield *key)
{
register char *ptr = line->text, *lim = ptr + line->length - 1;
register size_t eword = key->eword, echar = key->echar;
@@ -842,10 +1055,10 @@
`beginning' is the first character following the delimiting TAB.
Otherwise, leave PTR pointing at the first `blank' character after
the preceding field. */
- if (tab != TAB_DEFAULT)
+ if (tab[0] != TAB_DEFAULT)
while (ptr < lim && eword--)
{
- while (ptr < lim && *ptr != tab)
+ while (ptr < lim && *ptr != tab[0])
++ptr;
if (ptr < lim && (eword | echar))
++ptr;
@@ -891,10 +1104,10 @@
*/
/* Make LIM point to the end of (one byte past) the current field. */
- if (tab != TAB_DEFAULT)
+ if (tab[0] != TAB_DEFAULT)
{
char *newlim;
- newlim = memchr (ptr, tab, lim - ptr);
+ newlim = memchr (ptr, tab[0], lim - ptr);
if (newlim)
lim = newlim;
}
@@ -926,15 +1139,137 @@
return ptr;
}
+#if HAVE_MBRTOWC
+static char *
+limfield_mb (const struct line *line, const struct keyfield *key)
+{
+ char *ptr = line->text, *lim = ptr + line->length - 1;
+ size_t eword = key->eword, echar = key->echar;
+ int i;
+ size_t mblength;
+ mbstate_t state;
+
+ memset (&state, '\0', sizeof(mbstate_t));
+
+ if (tab[0])
+ while (ptr < lim && eword--)
+ {
+ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ ptr += mblength;
+ }
+ if (ptr < lim && (eword | echar))
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ ptr += mblength;
+ }
+ }
+ else
+ while (ptr < lim && eword--)
+ {
+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
+ ptr += mblength;
+ if (ptr < lim)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ ptr += mblength;
+ }
+ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength))
+ ptr += mblength;
+ }
+
+
+# ifdef POSIX_UNSPECIFIED
+ /* Make LIM point to the end of (one byte past) the current field. */
+ if (tab[0])
+ {
+ char *newlim, *p;
+
+ newlim = NULL;
+ for (p = ptr; p < lim;)
+ {
+ if (memcmp (p, tab, tab_length) == 0)
+ {
+ newlim = p;
+ break;
+ }
+
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ p += mblength;
+ }
+ }
+ else
+ {
+ char *newlim;
+ newlim = ptr;
+
+ while (newlim < lim && ismbblank (newlim, lim - newlim, &mblength))
+ newlim += mblength;
+ if (ptr < lim)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+ ptr += mblength;
+ }
+ while (newlim < lim && !ismbblank (newlim, lim - newlim, &mblength))
+ newlim += mblength;
+ lim = newlim;
+ }
+# endif
+
+ /* If we're skipping leading blanks, don't start counting characters
+ * until after skipping past any leading blanks. */
+ if (key->skipsblanks)
+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
+ ptr += mblength;
+
+ memset (&state, '\0', sizeof(mbstate_t));
+
+ /* Advance PTR by ECHAR (if possible), but no further than LIM. */
+ for (i = 0; i < echar; i++)
+ {
+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
+
+ if (ptr + mblength > lim)
+ break;
+ else
+ ptr += mblength;
+ }
+
+ return ptr;
+}
+#endif
+
/* Return the number of trailing blanks in FIELD, with LEN bytes. */
static size_t
trailing_blanks (char const *field, size_t len)
{
- size_t i;
- for (i = len; 0 < i && blanks[UCHAR (field[i - 1])]; i--)
- continue;
- return len - i;
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ size_t blanks = 0;
+
+ while (len) {
+ size_t mblength;
+ if (ismbblank (field, len, &mblength))
+ blanks++;
+ else
+ blanks = 0;
+
+ field += mblength, len -= mblength;
+ }
+
+ return blanks;
+ }
+ else
+#endif
+ {
+ size_t i;
+ for (i = len; 0 < i && blanks[UCHAR (field[i - 1])]; i--)
+ continue;
+ return len - i;
+ }
}
/* Fill BUF reading from FP, moving buf->left bytes from the end
@@ -1019,8 +1354,22 @@
else
{
if (key->skipsblanks)
- while (blanks[UCHAR (*line_start)])
- line_start++;
+#if HAVE_MBRTOWC
+ {
+ if (MB_CUR_MAX > 1)
+ {
+ size_t mblength;
+
+ while (ismbblank (line_start, ptr - line_start, &mblength))
+ line_start += mblength;
+ }
+ else
+#endif
+ {
+ while (blanks[UCHAR (*line_start)])
+ line_start++;
+ }
+ }
line->keybeg = line_start;
}
if (key->skipeblanks)
@@ -1128,13 +1477,32 @@
register int tmpa, tmpb, tmp;
register size_t log_a, log_b;
- tmpa = *a;
- tmpb = *b;
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ size_t mblength;
+ size_t alen = strnlen (a, MB_LEN_MAX);
+ size_t blen = strnlen (b, MB_LEN_MAX);
+
+ while (ismbblank (a, alen, &mblength))
+ a += mblength, alen -= mblength;
+ while (ismbblank (b, blen, &mblength))
+ b += mblength, blen -= mblength;
- while (blanks[UCHAR (tmpa)])
- tmpa = *++a;
- while (blanks[UCHAR (tmpb)])
- tmpb = *++b;
+ tmpa = *a;
+ tmpb = *b;
+ }
+ else
+#endif
+ {
+ tmpa = *a;
+ tmpb = *b;
+
+ while (blanks[UCHAR (tmpa)])
+ tmpa = *++a;
+ while (blanks[UCHAR (tmpb)])
+ tmpb = *++b;
+ }
if (tmpa == NEGATION_SIGN)
{
@@ -1268,15 +1636,60 @@
/* FIXME: maybe add option to try expensive FP conversion
only if A and B can't be compared more cheaply/accurately. */
- char *ea;
- char *eb;
- double a = strtod (sa, &ea);
- double b = strtod (sb, &eb);
+ char *bufa, *ea;
+ char *bufb, *eb;
+ double a;
+ double b;
+
+ char *p;
+ struct lconv *lconvp = localeconv ();
+ size_t thousands_sep_len = strlen (lconvp->thousands_sep);
+
+ bufa = (char *) xmalloc (strlen (sa) + 1);
+ bufb = (char *) xmalloc (strlen (sb) + 1);
+ strcpy (bufa, sa);
+ strcpy (bufb, sb);
+
+ if (force_general_numcompare)
+ {
+ while (1)
+ {
+ a = strtod (bufa, &ea);
+ if (memcmp (ea, lconvp->thousands_sep, thousands_sep_len) == 0)
+ {
+ for (p = ea; *(p + thousands_sep_len) != '\0'; p++)
+ *p = *(p + thousands_sep_len);
+ *p = '\0';
+ continue;
+ }
+ break;
+ }
+
+ while (1)
+ {
+ b = strtod (bufb, &eb);
+ if (memcmp (eb, lconvp->thousands_sep, thousands_sep_len) == 0)
+ {
+ for (p = eb; *(p + thousands_sep_len) != '\0'; p++)
+ *p = *(p + thousands_sep_len);
+ *p = '\0';
+ continue;
+ }
+ break;
+ }
+ }
+ else
+ {
+ a = strtod (bufa, &ea);
+ b = strtod (bufb, &eb);
+ }
/* Put conversion errors at the start of the collating sequence. */
- if (sa == ea)
- return sb == eb ? 0 : -1;
- if (sb == eb)
+ free (bufa);
+ free (bufb);
+ if (bufa == ea)
+ return bufb == eb ? 0 : -1;
+ if (bufb == eb)
return 1;
/* Sort numbers in the usual way, where -0 == +0. Put NaNs after
@@ -1294,7 +1707,7 @@
Return 0 if the name in S is not recognized. */
static int
-getmonth (const char *s, size_t len)
+getmonth_uni (const char *s, size_t len)
{
char *month;
register size_t i;
@@ -1332,11 +1745,79 @@
return result;
}
+#if HAVE_MBRTOWC
+static int
+getmonth_mb (const char *s, size_t len)
+{
+ char *month;
+ register size_t i;
+ register int lo = 0, hi = MONTHS_PER_YEAR, result;
+ char *tmp;
+ size_t wclength, mblength;
+ const char **pp;
+ const wchar_t **wpp;
+ wchar_t *month_wcs;
+ mbstate_t state;
+
+ while (len > 0 && ismbblank (s, len, &mblength))
+ {
+ s += mblength;
+ len -= mblength;
+ }
+
+ if (len == 0)
+ return 0;
+
+ month = (char *) alloca (len + 1);
+
+ tmp = (char *) alloca (len + 1);
+ memcpy (tmp, s, len);
+ tmp[len] = '\0';
+ pp = (const char **)&tmp;
+ month_wcs = (wchar_t *) alloca ((len + 1) * sizeof (wchar_t));
+ memset (&state, '\0', sizeof(mbstate_t));
+
+ wclength = mbsrtowcs (month_wcs, pp, len + 1, &state);
+ assert (wclength != (size_t)-1 && *pp == NULL);
+
+ for (i = 0; i < wclength; i++)
+ {
+ month_wcs[i] = towupper(month_wcs[i]);
+ if (iswblank (month_wcs[i]))
+ {
+ month_wcs[i] = L'\0';
+ break;
+ }
+ }
+
+ wpp = (const wchar_t **)&month_wcs;
+
+ mblength = wcsrtombs (month, wpp, len + 1, &state);
+ assert (mblength != (-1) && *wpp == NULL);
+
+ do
+ {
+ int ix = (lo + hi) / 2;
+
+ if (strncmp (month, monthtab[ix].name, strlen (monthtab[ix].name)) < 0)
+ hi = ix;
+ else
+ lo = ix;
+ }
+ while (hi - lo > 1);
+
+ result = (!strncmp (month, monthtab[lo].name, strlen (monthtab[lo].name))
+ ? monthtab[lo].val : 0);
+
+ return result;
+}
+#endif
+
/* Compare two lines A and B trying every key in sequence until there
are no more keys or a difference is found. */
static int
-keycompare (const struct line *a, const struct line *b)
+keycompare_uni (const struct line *a, const struct line *b)
{
struct keyfield const *key = keylist;
@@ -1507,6 +1988,187 @@
return key->reverse ? -diff : diff;
}
+#if HAVE_MBRTOWC
+static int
+keycompare_mb (const struct line *a, const struct line *b)
+{
+ struct keyfield *key = keylist;
+
+ /* For the first iteration only, the key positions have been
+ precomputed for us. */
+ char *texta = a->keybeg;
+ char *textb = b->keybeg;
+ char *lima = a->keylim;
+ char *limb = b->keylim;
+
+ size_t mblength_a, mblength_b;
+ wchar_t wc_a, wc_b;
+ mbstate_t state_a, state_b;
+
+ int diff;
+
+ memset (&state_a, '\0', sizeof(mbstate_t));
+ memset (&state_b, '\0', sizeof(mbstate_t));
+
+ for (;;)
+ {
+ unsigned char *translate = (unsigned char *) key->translate;
+ bool const *ignore = key->ignore;
+
+ /* Find the lengths. */
+ size_t lena = lima <= texta ? 0 : lima - texta;
+ size_t lenb = limb <= textb ? 0 : limb - textb;
+
+ if (key->skipeblanks)
+ {
+ char *a_end = texta + lena;
+ char *b_end = textb + lenb;
+ a_end -= trailing_blanks (texta, lena);
+ b_end -= trailing_blanks (textb, lenb);
+ lena = a_end - texta;
+ lenb = b_end - textb;
+ }
+
+ /* Actually compare the fields. */
+ if (key->numeric | key->general_numeric)
+ {
+ char savea = *lima, saveb = *limb;
+
+ *lima = *limb = '\0';
+ if (force_general_numcompare)
+ diff = general_numcompare (texta, textb);
+ else
+ diff = ((key->numeric ? numcompare : general_numcompare)
+ (texta, textb));
+ *lima = savea, *limb = saveb;
+ }
+ else if (key->month)
+ diff = getmonth (texta, lena) - getmonth (textb, lenb);
+ else
+ {
+ if (ignore || translate)
+ {
+ char *copy_a = (char *) alloca (lena + 1 + lenb + 1);
+ char *copy_b = copy_a + lena + 1;
+ size_t new_len_a, new_len_b;
+ size_t i, j;
+
+ /* Ignore and/or translate chars before comparing. */
+# define IGNORE_CHARS(NEW_LEN, LEN, TEXT, COPY, WC, MBLENGTH, STATE) \
+ do \
+ { \
+ wchar_t uwc; \
+ char mbc[MB_LEN_MAX]; \
+ mbstate_t state_wc; \
+ \
+ for (NEW_LEN = i = 0; i < LEN;) \
+ { \
+ mbstate_t state_bak; \
+ \
+ state_bak = STATE; \
+ MBLENGTH = mbrtowc (&WC, TEXT + i, LEN - i, &STATE); \
+ \
+ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1 \
+ || MBLENGTH == 0) \
+ { \
+ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1) \
+ STATE = state_bak; \
+ if (!ignore) \
+ COPY[NEW_LEN++] = TEXT[i++]; \
+ continue; \
+ } \
+ \
+ if (ignore) \
+ { \
+ if ((ignore == nonprinting && !iswprint (WC)) \
+ || (ignore == nondictionary \
+ && !iswalnum (WC) && !iswblank (WC))) \
+ { \
+ i += MBLENGTH; \
+ continue; \
+ } \
+ } \
+ \
+ if (translate) \
+ { \
+ \
+ uwc = toupper(WC); \
+ if (WC == uwc) \
+ { \
+ memcpy (mbc, TEXT + i, MBLENGTH); \
+ i += MBLENGTH; \
+ } \
+ else \
+ { \
+ i += MBLENGTH; \
+ WC = uwc; \
+ memset (&state_wc, '\0', sizeof (mbstate_t)); \
+ \
+ MBLENGTH = wcrtomb (mbc, WC, &state_wc); \
+ assert (MBLENGTH != (size_t)-1 && MBLENGTH != 0); \
+ } \
+ \
+ for (j = 0; j < MBLENGTH; j++) \
+ COPY[NEW_LEN++] = mbc[j]; \
+ } \
+ else \
+ for (j = 0; j < MBLENGTH; j++) \
+ COPY[NEW_LEN++] = TEXT[i++]; \
+ } \
+ COPY[NEW_LEN] = '\0'; \
+ } \
+ while (0)
+ IGNORE_CHARS (new_len_a, lena, texta, copy_a,
+ wc_a, mblength_a, state_a);
+ IGNORE_CHARS (new_len_b, lenb, textb, copy_b,
+ wc_b, mblength_b, state_b);
+ diff = xmemcoll (copy_a, new_len_a, copy_b, new_len_b);
+ }
+ else if (lena == 0)
+ diff = - NONZERO (lenb);
+ else if (lenb == 0)
+ goto greater;
+ else
+ diff = xmemcoll (texta, lena, textb, lenb);
+ }
+
+ if (diff)
+ goto not_equal;
+
+ key = key->next;
+ if (! key)
+ break;
+
+ /* Find the beginning and limit of the next field. */
+ if (key->eword != -1)
+ lima = limfield (a, key), limb = limfield (b, key);
+ else
+ lima = a->text + a->length - 1, limb = b->text + b->length - 1;
+
+ if (key->sword != -1)
+ texta = begfield (a, key), textb = begfield (b, key);
+ else
+ {
+ texta = a->text, textb = b->text;
+ if (key->skipsblanks)
+ {
+ while (texta < lima && ismbblank (texta, lima - texta, &mblength_a))
+ texta += mblength_a;
+ while (textb < limb && ismbblank (textb, limb - textb, &mblength_b))
+ textb += mblength_b;
+ }
+ }
+ }
+
+ return 0;
+
+greater:
+ diff = 1;
+not_equal:
+ return key->reverse ? -diff : diff;
+}
+#endif
+
/* Compare two lines A and B, returning negative, zero, or positive
depending on whether A compares less than, equal to, or greater than B. */
@@ -2252,20 +2914,44 @@
{
struct lconv const *lconvp = localeconv ();
- /* If the locale doesn't define a decimal point, or if the decimal
- point is multibyte, use the C decimal point. We don't support
- multibyte decimal points yet. */
decimal_point = *lconvp->decimal_point;
if (! decimal_point || lconvp->decimal_point[1])
- decimal_point = C_DECIMAL_POINT;
+ {
+ decimal_point = C_DECIMAL_POINT;
+ if (lconvp->decimal_point[0] && lconvp->decimal_point[1])
+ force_general_numcompare = 1;
+ }
/* We don't support multibyte thousands separators yet. */
th_sep = *lconvp->thousands_sep;
if (! th_sep || lconvp->thousands_sep[1])
- th_sep = CHAR_MAX + 1;
+ {
+ th_sep = CHAR_MAX + 1;
+ if (lconvp->thousands_sep[0] && lconvp->thousands_sep[1])
+ force_general_numcompare = 1;
+ }
}
#endif
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ inittables = inittables_mb;
+ begfield = begfield_mb;
+ limfield = limfield_mb;
+ getmonth = getmonth_mb;
+ keycompare = keycompare_mb;
+ }
+ else
+#endif
+ {
+ inittables = inittables_uni;
+ begfield = begfield_uni;
+ limfield = limfield_uni;
+ keycompare = keycompare_uni;
+ getmonth = getmonth_uni;
+ }
+
have_read_stdin = false;
inittables ();
@@ -2462,13 +3148,47 @@
case 't':
{
- int newtab = optarg[0];
- if (! newtab)
+ char newtab[MB_LEN_MAX + 1];
+ strncpy (newtab, optarg, MB_LEN_MAX);
+ if (! newtab[0])
error (SORT_FAILURE, 0, _("empty tab"));
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ wchar_t wc;
+ mbstate_t state;
+ size_t newtab_length, i;
+
+ memset (&state, '\0', sizeof (mbstate_t));
+ newtab_length = mbrtowc (&wc, newtab, strnlen (newtab, MB_LEN_MAX), &state);
+ switch (newtab_length)
+ {
+ case (size_t) -1:
+ case (size_t) -2:
+ case 0:
+ newtab_length = 1;
+ }
+
+ if (optarg[tab_length])
+ {
+ /* Provoke with `sort -txx'. Complain about
+ "multi-character tab" instead of "multibyte tab", so
+ that the diagnostic's wording does not need to be
+ changed once multibyte characters are supported. */
+ error (SORT_FAILURE, 0, _("multi-character tab `%s'"),
+ optarg);
+ }
+
+ for (i = 0; i < newtab_length; i++)
+ tab[i] = newtab[i];
+ }
+ else
+#endif
+
if (optarg[1])
{
if (strcmp (optarg, "\\0") == 0)
- newtab = '\0';
+ newtab[0] = '\0';
else
{
/* Provoke with `sort -txx'. Complain about
@@ -2479,9 +3199,9 @@
optarg);
}
}
- if (tab != TAB_DEFAULT && tab != newtab)
+ if (tab[0] != TAB_DEFAULT && tab[0] != newtab[0])
error (SORT_FAILURE, 0, _("incompatible tabs"));
- tab = newtab;
+ tab[0] = newtab[0];
}
break;
--- coreutils-5.1.3/src/uniq.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/uniq.c 2004-02-16 15:36:40.000000000 +0000
@@ -23,6 +23,16 @@
#include <getopt.h>
#include <sys/types.h>
+/* Get mbstate_t, mbrtowc(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
+/* Get isw* functions. */
+#if HAVE_WCTYPE_H
+# include <wctype.h>
+#endif
+
#include "system.h"
#include "argmatch.h"
#include "linebuffer.h"
@@ -31,7 +41,19 @@
#include "posixver.h"
#include "xmemcoll.h"
#include "xstrtol.h"
-#include "memcasecmp.h"
+#include "xmemcoll.h"
+
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+ installation; work around this configuration error. */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "uniq"
@@ -108,6 +130,10 @@
/* Select whether/how to delimit groups of duplicate lines. */
static enum delimit_method delimit_groups;
+/* Function pointers. */
+static char *
+(*find_field) (struct linebuffer *line);
+
static struct option const longopts[] =
{
{"count", no_argument, NULL, 'c'},
@@ -188,7 +214,7 @@
return a pointer to the beginning of the line's field to be compared. */
static char *
-find_field (const struct linebuffer *line)
+find_field_uni (struct linebuffer *line)
{
register size_t count;
register char *lp = line->buffer;
@@ -209,6 +235,83 @@
return lp + i;
}
+#if HAVE_MBRTOWC
+
+# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \
+ do \
+ { \
+ mbstate_t state_bak; \
+ \
+ CONVFAIL = 0; \
+ state_bak = *STATEP; \
+ \
+ MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \
+ \
+ switch (MBLENGTH) \
+ { \
+ case (size_t)-2: \
+ case (size_t)-1: \
+ *STATEP = state_bak; \
+ CONVFAIL++; \
+ /* Fall through */ \
+ case 0: \
+ MBLENGTH = 1; \
+ } \
+ } \
+ while (0)
+
+static char *
+find_field_multi (struct linebuffer *line)
+{
+ size_t count;
+ char *lp = line->buffer;
+ size_t size = line->length - 1;
+ size_t pos;
+ size_t mblength;
+ wchar_t wc;
+ mbstate_t *statep;
+ int convfail;
+
+ pos = 0;
+ statep = &(line->state);
+
+ /* skip fields. */
+ for (count = 0; count < skip_fields && pos < size; count++)
+ {
+ while (pos < size)
+ {
+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
+
+ if (convfail || !iswblank (wc))
+ {
+ pos += mblength;
+ break;
+ }
+ pos += mblength;
+ }
+
+ while (pos < size)
+ {
+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
+
+ if (!convfail && iswblank (wc))
+ break;
+
+ pos += mblength;
+ }
+ }
+
+ /* skip fields. */
+ for (count = 0; count < skip_chars && pos < size; count++)
+ {
+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
+ pos += mblength;
+ }
+
+ return lp + pos;
+}
+#endif
+
/* Return zero if two strings OLD and NEW match, nonzero if not.
OLD and NEW point not to the beginnings of the lines
but rather to the beginnings of the fields to compare.
@@ -217,6 +320,8 @@
static int
different (char *old, char *new, size_t oldlen, size_t newlen)
{
+ char *copy_old, *copy_new;
+
if (check_chars < oldlen)
oldlen = check_chars;
if (check_chars < newlen)
@@ -224,15 +329,93 @@
if (ignore_case)
{
- /* FIXME: This should invoke strcoll somehow. */
- return oldlen != newlen || memcasecmp (old, new, oldlen);
+ size_t i;
+
+ copy_old = alloca (oldlen + 1);
+ copy_new = alloca (oldlen + 1);
+
+ for (i = 0; i < oldlen; i++)
+ {
+ copy_old[i] = toupper (old[i]);
+ copy_new[i] = toupper (new[i]);
+ }
}
- else if (HAVE_SETLOCALE && hard_LC_COLLATE)
- return xmemcoll (old, oldlen, new, newlen);
else
- return oldlen != newlen || memcmp (old, new, oldlen);
+ {
+ copy_old = (char *)old;
+ copy_new = (char *)new;
+ }
+
+ return xmemcoll (copy_old, oldlen, copy_new, newlen);
}
+#if HAVE_MBRTOWC
+static int
+different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate)
+{
+ size_t i, j, chars;
+ const char *str[2];
+ char *copy[2];
+ size_t len[2];
+ mbstate_t state[2];
+ size_t mblength;
+ wchar_t wc, uwc;
+ mbstate_t state_bak;
+
+ str[0] = old;
+ str[1] = new;
+ len[0] = oldlen;
+ len[1] = newlen;
+ state[0] = oldstate;
+ state[1] = newstate;
+
+ for (i = 0; i < 2; i++)
+ {
+ copy[i] = alloca (len[i] + 1);
+
+ for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++)
+ {
+ state_bak = state[i];
+ mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i]));
+
+ switch (mblength)
+ {
+ case (size_t)-1:
+ case (size_t)-2:
+ state[i] = state_bak;
+ /* Fall through */
+ case 0:
+ mblength = 1;
+ break;
+
+ default:
+ if (ignore_case)
+ {
+ uwc = towupper (wc);
+
+ if (uwc != wc)
+ {
+ mbstate_t state_wc;
+
+ memset (&state_wc, '\0', sizeof(mbstate_t));
+ wcrtomb (copy[i] + j, uwc, &state_wc);
+ }
+ else
+ memcpy (copy[i] + j, str[i] + j, mblength);
+ }
+ else
+ memcpy (copy[i] + j, str[i] + j, mblength);
+ }
+ j += mblength;
+ }
+ copy[i][j] = '\0';
+ len[i] = j;
+ }
+
+ return xmemcoll (copy[0], len[0], copy[1], len[1]);
+}
+#endif
+
/* Output the line in linebuffer LINE to stream STREAM
provided that the switches say it should be output.
MATCH is true if the line matches the previous line.
@@ -296,15 +479,43 @@
{
char *prevfield IF_LINT (= NULL);
size_t prevlen IF_LINT (= 0);
+#if HAVE_MBRTOWC
+ mbstate_t prevstate;
+
+ memset (&prevstate, '\0', sizeof (mbstate_t));
+#endif
while (!feof (istream))
{
char *thisfield;
size_t thislen;
+#if HAVE_MBRTOWC
+ mbstate_t thisstate;
+#endif
+
if (readlinebuffer (thisline, istream) == 0)
break;
thisfield = find_field (thisline);
thislen = thisline->length - 1 - (thisfield - thisline->buffer);
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ thisstate = thisline->state;
+
+ if (prevline->length == 0 || different_multi
+ (thisfield, prevfield, thislen, prevlen, thisstate, prevstate))
+ {
+ fwrite (thisline->buffer, sizeof (char),
+ thisline->length, ostream);
+
+ SWAP_LINES (prevline, thisline);
+ prevfield = thisfield;
+ prevlen = thislen;
+ prevstate = thisstate;
+ }
+ }
+ else
+#endif
if (prevline->length == 0
|| different (thisfield, prevfield, thislen, prevlen))
{
@@ -323,17 +534,26 @@
size_t prevlen;
int match_count = 0;
int first_delimiter = 1;
+#if HAVE_MBRTOWC
+ mbstate_t prevstate;
+#endif
if (readlinebuffer (prevline, istream) == 0)
goto closefiles;
prevfield = find_field (prevline);
prevlen = prevline->length - 1 - (prevfield - prevline->buffer);
+#if HAVE_MBRTOWC
+ prevstate = prevline->state;
+#endif
while (!feof (istream))
{
bool match;
char *thisfield;
size_t thislen;
+#if HAVE_MBRTOWC
+ mbstate_t thisstate;
+#endif
if (readlinebuffer (thisline, istream) == 0)
{
if (ferror (istream))
@@ -342,6 +562,15 @@
}
thisfield = find_field (thisline);
thislen = thisline->length - 1 - (thisfield - thisline->buffer);
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ thisstate = thisline->state;
+ match = !different_multi (thisfield, prevfield,
+ thislen, prevlen, thisstate, prevstate);
+ }
+ else
+#endif
match = !different (thisfield, prevfield, thislen, prevlen);
if (match)
@@ -369,6 +598,9 @@
SWAP_LINES (prevline, thisline);
prevfield = thisfield;
prevlen = thislen;
+#if HAVE_MBRTOWC
+ prevstate = thisstate;
+#endif
if (!match)
match_count = 0;
}
@@ -411,6 +643,19 @@
atexit (close_stdout);
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ find_field = find_field_multi;
+ }
+ else
+#endif
+ {
+ find_field = find_field_uni;
+ }
+
+
+
skip_chars = 0;
skip_fields = 0;
check_chars = SIZE_MAX;
--- coreutils-5.1.3/src/expand.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/expand.c 2004-02-16 15:36:40.000000000 +0000
@@ -38,12 +38,29 @@
#include <stdio.h>
#include <getopt.h>
#include <sys/types.h>
+
+/* Get mbstate_t, mbrtowc(), wcwidth(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
#include "system.h"
#include "error.h"
#include "posixver.h"
#include "quote.h"
#include "xstrndup.h"
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+ installation; work around this configuration error. */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "expand"
@@ -332,6 +349,7 @@
++column;
}
}
+
else
{
if (convert)
@@ -343,7 +361,8 @@
}
else
{
- ++column;
+ if (!ISCNTRL (c))
+ ++column;
if (convert_entire_line == 0)
convert = 0;
}
@@ -353,6 +372,146 @@
}
}
+#if HAVE_MBRTOWC
+static void
+expand_multibyte (void)
+{
+ FILE *fp; /* Input strem. */
+ mbstate_t i_state; /* Current shift state of the input stream. */
+ mbstate_t i_state_bak; /* Back up the I_STATE. */
+ mbstate_t o_state; /* Current shift state of the output stream. */
+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
+ char *bufpos; /* Next read position of BUF. */
+ size_t buflen = 0; /* The length of the byte sequence in buf. */
+ wchar_t wc; /* A gotten wide character. */
+ size_t mblength; /* The byte size of a multibyte character
+ which shows as same character as WC. */
+ int tab_index = 0; /* Index in `tab_list' of next tabstop. */
+ int column = 0; /* Column on screen of the next char. */
+ int next_tab_column; /* Column the next tab stop is on. */
+ int convert = 1; /* If nonzero, perform translations. */
+
+ fp = next_file ((FILE *) NULL);
+ if (fp == NULL)
+ return;
+
+ /* Binary I/O will preserve the original EOL style (DOS/Unix) of files. */
+ SET_BINARY2 (fileno (fp), STDOUT_FILENO);
+
+ memset (&o_state, '\0', sizeof(mbstate_t));
+ memset (&i_state, '\0', sizeof(mbstate_t));
+
+ for (;;)
+ {
+ /* Refill the buffer BUF. */
+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp))
+ {
+ memmove (buf, bufpos, buflen);
+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp);
+ bufpos = buf;
+ }
+
+ /* No character is left in BUF. */
+ if (buflen < 1)
+ {
+ fp = next_file (fp);
+
+ if (fp == NULL)
+ break; /* No more files. */
+ else
+ {
+ memset (&i_state, '\0', sizeof(mbstate_t));
+ SET_BINARY2 (fileno (fp), STDOUT_FILENO);
+ continue;
+ }
+ }
+
+ /* Get a wide character. */
+ i_state_bak = i_state;
+ mblength = mbrtowc (&wc, bufpos, buflen, &i_state);
+
+ switch (mblength)
+ {
+ case (size_t)-1: /* illegal byte sequence. */
+ case (size_t)-2:
+ mblength = 1;
+ i_state = i_state_bak;
+ if (convert)
+ {
+ ++column;
+ if (convert_entire_line == 0)
+ convert = 0;
+ }
+ putchar (*bufpos);
+ break;
+
+ case 0: /* null. */
+ mblength = 1;
+ if (convert && convert_entire_line == 0)
+ convert = 0;
+ putchar ('\0');
+ break;
+
+ default:
+ if (wc == L'\n') /* LF. */
+ {
+ tab_index = 0;
+ column = 0;
+ convert = 1;
+ putchar ('\n');
+ }
+ else if (wc == L'\t' && convert) /* Tab. */
+ {
+ if (tab_size == 0)
+ {
+ /* Do not let tab_index == first_free_tab;
+ stop when it is 1 less. */
+ while (tab_index < first_free_tab - 1
+ && column >= tab_list[tab_index])
+ tab_index++;
+ next_tab_column = tab_list[tab_index];
+ if (tab_index < first_free_tab - 1)
+ tab_index++;
+ if (column >= next_tab_column)
+ next_tab_column = column + 1;
+ }
+ else
+ next_tab_column = column + tab_size - column % tab_size;
+
+ while (column < next_tab_column)
+ {
+ putchar (' ');
+ ++column;
+ }
+ }
+ else /* Others. */
+ {
+ if (convert)
+ {
+ if (wc == L'\b')
+ {
+ if (column > 0)
+ --column;
+ }
+ else
+ {
+ int width; /* The width of WC. */
+
+ width = wcwidth (wc);
+ column += (width > 0) ? width : 0;
+ if (convert_entire_line == 0)
+ convert = 0;
+ }
+ }
+ fwrite (bufpos, sizeof(char), mblength, stdout);
+ }
+ }
+ buflen -= mblength;
+ bufpos += mblength;
+ }
+}
+#endif
+
int
main (int argc, char **argv)
{
@@ -424,7 +583,12 @@
file_list = (optind < argc ? &argv[optind] : stdin_argv);
- expand ();
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ expand_multibyte ();
+ else
+#endif
+ expand ();
if (have_read_stdin && fclose (stdin) == EOF)
error (EXIT_FAILURE, errno, "-");
--- coreutils-5.1.3/src/fold.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/fold.c 2004-02-16 15:36:40.000000000 +0000
@@ -23,31 +23,74 @@
#include <getopt.h>
#include <sys/types.h>
+/* Get mbstate_t, mbrtowc(), wcwidth(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
+/* Get iswprint(), iswblank(), wcwidth(). */
+#if HAVE_WCTYPE_H
+# include <wctype.h>
+#endif
+
#include "system.h"
#include "error.h"
#include "posixver.h"
#include "xstrtol.h"
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+ installation; work around this configuration error. */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# undef MB_LEN_MAX
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "fold"
#define AUTHORS "David MacKenzie"
+#define FATAL_ERROR(Message) \
+ do \
+ { \
+ error (0, 0, (Message)); \
+ usage (2); \
+ } \
+ while (0)
+
+enum operating_mode
+{
+ /* Fold texts by columns that are at the given positions. */
+ column_mode,
+
+ /* Fold texts by bytes that are at the given positions. */
+ byte_mode,
+
+ /* Fold texts by characters that are at the given positions. */
+ character_mode,
+};
+
/* The name this program was run with. */
char *program_name;
+/* The argument shows current mode. (Default: column_mode) */
+static enum operating_mode operating_mode;
+
/* If nonzero, try to break on whitespace. */
static bool break_spaces;
-/* If nonzero, count bytes, not column positions. */
-static bool count_bytes;
-
/* If nonzero, at least one of the files we read was standard input. */
static bool have_read_stdin;
static struct option const longopts[] =
{
{"bytes", no_argument, NULL, 'b'},
+ {"characters", no_argument, NULL, 'c'},
{"spaces", no_argument, NULL, 's'},
{"width", required_argument, NULL, 'w'},
{GETOPT_HELP_OPTION_DECL},
@@ -77,6 +120,7 @@
"), stdout);
fputs (_("\
-b, --bytes count bytes rather than columns\n\
+ -c, --characters count characters rather than columns\n\
-s, --spaces break at spaces\n\
-w, --width=WIDTH use WIDTH columns instead of 80\n\
"), stdout);
@@ -94,7 +138,7 @@
static size_t
adjust_column (size_t column, char c)
{
- if (!count_bytes)
+ if (operating_mode != byte_mode)
{
if (c == '\b')
{
@@ -113,34 +157,14 @@
return column;
}
-/* Fold file FILENAME, or standard input if FILENAME is "-",
- to stdout, with maximum line length WIDTH.
- Return 0 if successful, 1 if an error occurs. */
-
static int
-fold_file (char *filename, int width)
+fold_text (FILE *istream, int width, int *saved_errno)
{
- FILE *istream;
register int c;
size_t column = 0; /* Screen column where next char will go. */
size_t offset_out = 0; /* Index in `line_out' for next char. */
static char *line_out = NULL;
static size_t allocated_out = 0;
- int saved_errno;
-
- if (STREQ (filename, "-"))
- {
- istream = stdin;
- have_read_stdin = true;
- }
- else
- istream = fopen (filename, "r");
-
- if (istream == NULL)
- {
- error (0, errno, "%s", filename);
- return 1;
- }
while ((c = getc (istream)) != EOF)
{
@@ -168,6 +192,15 @@
bool found_blank = false;
size_t logical_end = offset_out;
+ /* If LINE_OUT has no wide character,
+ put a new wide character in LINE_OUT
+ if column is bigger than width. */
+ if (offset_out == 0)
+ {
+ line_out[offset_out++] = c;
+ continue;
+ }
+
/* Look for the last blank. */
while (logical_end)
{
@@ -214,11 +247,225 @@
line_out[offset_out++] = c;
}
- saved_errno = errno;
+ *saved_errno = errno;
if (offset_out)
fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
+ free(line_out);
+}
+
+#if HAVE_MBRTOWC
+static void
+fold_multibyte_text (FILE *istream, int width, int *saved_errno)
+{
+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
+ size_t buflen = 0; /* The length of the byte sequence in buf. */
+ char *bufpos; /* Next read position of BUF. */
+ wint_t wc; /* A gotten wide character. */
+ size_t mblength; /* The byte size of a multibyte character which shows
+ as same character as WC. */
+ mbstate_t state, state_bak; /* State of the stream. */
+ int convfail; /* 1, when conversion is failed. Otherwise 0. */
+
+ char *line_out = NULL;
+ size_t offset_out = 0; /* Index in `line_out' for next char. */
+ size_t allocated_out = 0;
+
+ int increment;
+ size_t column = 0;
+
+ size_t last_blank_pos;
+ size_t last_blank_column;
+ int is_blank_seen;
+ int last_blank_increment;
+ int is_bs_following_last_blank;
+ size_t bs_following_last_blank_num;
+ int is_cr_after_last_blank;
+
+#define CLEAR_FLAGS \
+ do \
+ { \
+ last_blank_pos = 0; \
+ last_blank_column = 0; \
+ is_blank_seen = 0; \
+ is_bs_following_last_blank = 0; \
+ bs_following_last_blank_num = 0; \
+ is_cr_after_last_blank = 0; \
+ } \
+ while (0)
+
+#define START_NEW_LINE \
+ do \
+ { \
+ putchar ('\n'); \
+ column = 0; \
+ offset_out = 0; \
+ CLEAR_FLAGS; \
+ } \
+ while (0)
+
+ CLEAR_FLAGS;
+ memset (&state, '\0', sizeof(mbstate_t));
+
+ for (;; bufpos += mblength, buflen -= mblength)
+ {
+ if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream))
+ {
+ memmove (buf, bufpos, buflen);
+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream);
+ bufpos = buf;
+ }
+
+ if (buflen < 1)
+ break;
+
+ /* Get a wide character. */
+ convfail = 0;
+ state_bak = state;
+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state);
+
+ switch (mblength)
+ {
+ case (size_t)-1:
+ case (size_t)-2:
+ convfail++;
+ state = state_bak;
+ /* Fall through. */
+
+ case 0:
+ mblength = 1;
+ break;
+ }
+
+rescan:
+ if (operating_mode == byte_mode) /* byte mode */
+ increment = mblength;
+ else if (operating_mode == character_mode) /* character mode */
+ increment = 1;
+ else /* column mode */
+ {
+ if (convfail)
+ increment = 1;
+ else
+ {
+ switch (wc)
+ {
+ case L'\n':
+ fwrite (line_out, sizeof(char), offset_out, stdout);
+ START_NEW_LINE;
+ continue;
+
+ case L'\b':
+ increment = (column > 0) ? -1 : 0;
+ break;
+
+ case L'\r':
+ increment = -1 * column;
+ break;
+
+ case L'\t':
+ increment = 8 - column % 8;
+ break;
+
+ default:
+ increment = wcwidth (wc);
+ increment = (increment < 0) ? 0 : increment;
+ }
+ }
+ }
+
+ if (column + increment > width && break_spaces && last_blank_pos)
+ {
+ fwrite (line_out, sizeof(char), last_blank_pos, stdout);
+ putchar ('\n');
+
+ offset_out = offset_out - last_blank_pos;
+ column = column - last_blank_column + ((is_cr_after_last_blank)
+ ? last_blank_increment : bs_following_last_blank_num);
+ memmove (line_out, line_out + last_blank_pos, offset_out);
+ CLEAR_FLAGS;
+ goto rescan;
+ }
+
+ if (column + increment > width && column != 0)
+ {
+ fwrite (line_out, sizeof(char), offset_out, stdout);
+ START_NEW_LINE;
+ goto rescan;
+ }
+
+ if (allocated_out < offset_out + mblength)
+ {
+ allocated_out += 1024;
+ line_out = xrealloc (line_out, allocated_out);
+ }
+
+ memcpy (line_out + offset_out, bufpos, mblength);
+ offset_out += mblength;
+ column += increment;
+
+ if (is_blank_seen && !convfail && wc == L'\r')
+ is_cr_after_last_blank = 1;
+
+ if (is_bs_following_last_blank && !convfail && wc == L'\b')
+ ++bs_following_last_blank_num;
+ else
+ is_bs_following_last_blank = 0;
+
+ if (break_spaces && !convfail && iswblank (wc))
+ {
+ last_blank_pos = offset_out;
+ last_blank_column = column;
+ is_blank_seen = 1;
+ last_blank_increment = increment;
+ is_bs_following_last_blank = 1;
+ bs_following_last_blank_num = 0;
+ is_cr_after_last_blank = 0;
+ }
+ }
+
+ *saved_errno = errno;
+
+ if (offset_out)
+ fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
+
+ free(line_out);
+}
+#endif
+
+/* Fold file FILENAME, or standard input if FILENAME is "-",
+ to stdout, with maximum line length WIDTH.
+ Return 0 if successful, 1 if an error occurs. */
+
+static int
+fold_file (char *filename, int width)
+{
+ FILE *istream;
+ int saved_errno;
+
+ if (STREQ (filename, "-"))
+ {
+ istream = stdin;
+ have_read_stdin = 1;
+ }
+ else
+ istream = fopen (filename, "r");
+
+ if (istream == NULL)
+ {
+ error (0, errno, "%s", filename);
+ return 1;
+ }
+
+ /* Define how ISTREAM is being folded. */
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ fold_multibyte_text (istream, width, &saved_errno);
+ else
+#endif
+ fold_text (istream, width, &saved_errno);
+
if (ferror (istream))
{
error (0, saved_errno, "%s", filename);
@@ -251,7 +498,8 @@
atexit (close_stdout);
- break_spaces = count_bytes = have_read_stdin = false;
+ operating_mode = column_mode;
+ break_spaces = have_read_stdin = false;
/* Turn any numeric options into -w options. */
for (i = 1; i < argc; i++)
@@ -278,7 +526,7 @@
}
}
- while ((optc = getopt_long (argc, argv, "bsw:", longopts, NULL)) != -1)
+ while ((optc = getopt_long (argc, argv, "bcsw:", longopts, NULL)) != -1)
{
switch (optc)
{
@@ -286,7 +534,15 @@
break;
case 'b': /* Count bytes rather than columns. */
- count_bytes = true;
+ if (operating_mode != column_mode)
+ FATAL_ERROR (_("only one way of folding may be specified"));
+ operating_mode = byte_mode;
+ break;
+
+ case 'c':
+ if (operating_mode != column_mode)
+ FATAL_ERROR (_("only one way of folding may be specified"));
+ operating_mode = character_mode;
break;
case 's': /* Break at word boundaries. */
--- coreutils-5.1.3/src/join.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/join.c 2004-02-16 15:36:40.000000000 +0000
@@ -24,16 +24,30 @@
#include <sys/types.h>
#include <getopt.h>
+/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
+/* Get iswblank(), towupper. */
+#if HAVE_WCTYPE_H
+# include <wctype.h>
+#endif
+
#include "system.h"
#include "error.h"
#include "hard-locale.h"
#include "linebuffer.h"
-#include "memcasecmp.h"
#include "posixver.h"
#include "quote.h"
#include "xmemcoll.h"
#include "xstrtol.h"
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "join"
@@ -110,7 +124,10 @@
/* Tab character separating fields; if this is NUL fields are separated
by any nonempty string of white space, otherwise by exactly one
tab character. */
-static char tab;
+static char *tab = NULL;
+
+/* The number of bytes used for tab. */
+static size_t tablen = 0;
/* When using getopt_long_only, no long option can start with
a character that is a short option. */
@@ -222,6 +239,8 @@
/* Fill in the `fields' structure in LINE. */
+/* Fill in the `fields' structure in LINE. */
+
static void
xfields (struct line *line)
{
@@ -231,9 +250,9 @@
if (ptr == lim)
return;
- if (tab)
+ if (tab != NULL)
{
- unsigned char t = tab;
+ unsigned char t = tab[0];
char *sep;
for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1)
extract_field (line, ptr, sep - ptr);
@@ -262,6 +281,147 @@
extract_field (line, ptr, lim - ptr);
}
+#if HAVE_MBRTOWC
+static void
+xfields_multibyte (struct line *line)
+{
+ char *ptr = line->buf.buffer;
+ char const *lim = ptr + line->buf.length - 1;
+ wchar_t wc = 0;
+ size_t mblength;
+ mbstate_t state, state_bak;
+
+ memset (&state, 0, sizeof (mbstate_t));
+
+ if (ptr == lim)
+ return;
+
+ if (tab != NULL)
+ {
+ unsigned char t = tab[0];
+ char *sep = ptr;
+ for (; ptr < lim; ptr = sep + 1)
+ {
+ while (sep < lim)
+ {
+ state_bak = state;
+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
+
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ mblength = 1;
+ state = state_bak;
+ }
+ mblength = (mblength < 1) ? 1 : mblength;
+
+ if (mblength == tablen && !memcmp (sep, tab, mblength))
+ break;
+ else
+ {
+ sep += mblength;
+ continue;
+ }
+ }
+
+ if (sep == lim)
+ break;
+
+ extract_field (line, ptr, sep - ptr);
+ }
+ }
+ else
+ {
+ /* Skip leading blanks before the first field. */
+ while(ptr < lim)
+ {
+ state_bak = state;
+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
+
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ mblength = 1;
+ state = state_bak;
+ break;
+ }
+ mblength = (mblength < 1) ? 1 : mblength;
+
+ if (!iswblank(wc))
+ break;
+ ptr += mblength;
+ }
+
+ do
+ {
+ char *sep;
+ state_bak = state;
+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ mblength = 1;
+ state = state_bak;
+ break;
+ }
+ mblength = (mblength < 1) ? 1 : mblength;
+
+ sep = ptr + mblength;
+ while (sep != lim)
+ {
+ state_bak = state;
+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ mblength = 1;
+ state = state_bak;
+ break;
+ }
+ mblength = (mblength < 1) ? 1 : mblength;
+
+ if (iswblank (wc))
+ break;
+
+ sep += mblength;
+ }
+
+ extract_field (line, ptr, sep - ptr);
+ if (sep == lim)
+ return;
+
+ state_bak = state;
+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ mblength = 1;
+ state = state_bak;
+ break;
+ }
+ mblength = (mblength < 1) ? 1 : mblength;
+
+ ptr = sep + mblength;
+ while (ptr != lim)
+ {
+ state_bak = state;
+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ mblength = 1;
+ state = state_bak;
+ break;
+ }
+ mblength = (mblength < 1) ? 1 : mblength;
+
+ if (!iswblank (wc))
+ break;
+
+ ptr += mblength;
+ }
+ }
+ while (ptr != lim);
+ }
+
+ extract_field (line, ptr, lim - ptr);
+}
+#endif
+
/* Read a line from FP into LINE and split it into fields.
Return true if successful. */
@@ -282,6 +442,11 @@
line->nfields_allocated = 0;
line->nfields = 0;
line->fields = NULL;
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ xfields_multibyte (line);
+ else
+#endif
xfields (line);
return true;
}
@@ -336,56 +501,116 @@
keycmp (struct line const *line1, struct line const *line2)
{
/* Start of field to compare in each file. */
- char *beg1;
- char *beg2;
-
- size_t len1;
- size_t len2; /* Length of fields to compare. */
+ char *beg[2];
+ char *copy[2];
+ size_t len[2]; /* Length of fields to compare. */
int diff;
+ int i, j;
if (join_field_1 < line1->nfields)
{
- beg1 = line1->fields[join_field_1].beg;
- len1 = line1->fields[join_field_1].len;
+ beg[0] = line1->fields[join_field_1].beg;
+ len[0] = line1->fields[join_field_1].len;
}
else
{
- beg1 = NULL;
- len1 = 0;
+ beg[0] = NULL;
+ len[0] = 0;
}
if (join_field_2 < line2->nfields)
{
- beg2 = line2->fields[join_field_2].beg;
- len2 = line2->fields[join_field_2].len;
+ beg[1] = line2->fields[join_field_2].beg;
+ len[1] = line2->fields[join_field_2].len;
}
else
{
- beg2 = NULL;
- len2 = 0;
+ beg[1] = NULL;
+ len[1] = 0;
}
- if (len1 == 0)
- return len2 == 0 ? 0 : -1;
- if (len2 == 0)
+ if (len[0] == 0)
+ return len[1] == 0 ? 0 : -1;
+ if (len[1] == 0)
return 1;
if (ignore_case)
{
- /* FIXME: ignore_case does not work with NLS (in particular,
- with multibyte chars). */
- diff = memcasecmp (beg1, beg2, MIN (len1, len2));
+#ifdef HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ size_t mblength;
+ wchar_t wc, uwc;
+ mbstate_t state, state_bak;
+
+ memset (&state, '\0', sizeof (mbstate_t));
+
+ for (i = 0; i < 2; i++)
+ {
+ copy[i] = alloca (len[i] + 1);
+
+ for (j = 0; j < MIN (len[0], len[1]);)
+ {
+ state_bak = state;
+ mblength = mbrtowc (&wc, beg[i] + j, len[i] - j, &state);
+
+ switch (mblength)
+ {
+ case (size_t) -1:
+ case (size_t) -2:
+ state = state_bak;
+ /* Fall through */
+ case 0:
+ mblength = 1;
+ break;
+
+ default:
+ uwc = towupper (wc);
+
+ if (uwc != wc)
+ {
+ mbstate_t state_wc;
+
+ memset (&state_wc, '\0', sizeof (mbstate_t));
+ wcrtomb (copy[i] + j, uwc, &state_wc);
+ }
+ else
+ memcpy (copy[i] + j, beg[i] + j, mblength);
+ }
+ j += mblength;
+ }
+ copy[i][j] = '\0';
+ }
+ }
+ else
+#endif
+ {
+ for (i = 0; i < 2; i++)
+ {
+ copy[i] = alloca (len[i] + 1);
+
+ for (j = 0; j < MIN (len[0], len[1]); j++)
+ copy[i][j] = toupper (beg[i][j]);
+
+ copy[i][j] = '\0';
+ }
+ }
}
else
{
- if (HAVE_SETLOCALE && hard_LC_COLLATE)
- return xmemcoll (beg1, len1, beg2, len2);
- diff = memcmp (beg1, beg2, MIN (len1, len2));
+ copy[0] = (unsigned char *) beg[0];
+ copy[1] = (unsigned char *) beg[1];
}
+#ifdef HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ return xmemcoll ((char *) copy[0], len[0], (char *) copy[1], len[1]);
+#endif
+ diff = memcmp (copy[0], copy[1], MIN (len[0], len[1]));
+
if (diff)
return diff;
- return len1 < len2 ? -1 : len1 != len2;
+ return len[0] - len[1];
}
/* Print field N of LINE if it exists and is nonempty, otherwise
@@ -410,11 +635,18 @@
/* Print the join of LINE1 and LINE2. */
+#define PUT_TAB_CHAR \
+ do \
+ { \
+ (tab != NULL) ? \
+ fwrite(tab, sizeof(char), tablen, stdout) : putchar (' '); \
+ } \
+ while (0)
+
static void
prjoin (struct line const *line1, struct line const *line2)
{
const struct outlist *outlist;
- char output_separator = tab ? tab : ' ';
outlist = outlist_head.next;
if (outlist)
@@ -430,12 +662,12 @@
if (o->file == 0)
{
if (line1 == &uni_blank)
- {
+ {
line = line2;
field = join_field_2;
}
else
- {
+ {
line = line1;
field = join_field_1;
}
@@ -449,7 +681,7 @@
o = o->next;
if (o == NULL)
break;
- putchar (output_separator);
+ PUT_TAB_CHAR;
}
putchar ('\n');
}
@@ -467,23 +699,23 @@
prfield (join_field_1, line1);
for (i = 0; i < join_field_1 && i < line1->nfields; ++i)
{
- putchar (output_separator);
+ PUT_TAB_CHAR;
prfield (i, line1);
}
for (i = join_field_1 + 1; i < line1->nfields; ++i)
{
- putchar (output_separator);
+ PUT_TAB_CHAR;
prfield (i, line1);
}
for (i = 0; i < join_field_2 && i < line2->nfields; ++i)
{
- putchar (output_separator);
+ PUT_TAB_CHAR;
prfield (i, line2);
}
for (i = join_field_2 + 1; i < line2->nfields; ++i)
{
- putchar (output_separator);
+ PUT_TAB_CHAR;
prfield (i, line2);
}
putchar ('\n');
@@ -678,7 +910,7 @@
valid = true;
}
else
- {
+ {
/* `0' must be all alone -- no `.FIELD'. */
error (0, 0, _("invalid field specifier: `%s'"), s);
}
@@ -817,7 +1049,23 @@
break;
case 't':
- tab = *optarg;
+ tab = xstrdup(optarg);
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ {
+ mbstate_t state;
+
+ memset(&state, 0, sizeof(mbstate_t));
+ tablen = mbrtowc(NULL, optarg, strnlen(optarg, MB_LEN_MAX), &state);
+ if (tablen == (size_t) 0
+ || tablen == (size_t)-1 || tablen == (size_t)-2)
+ tablen = 1;
+ }
+ else
+ {
+#endif
+ tablen = 1;
+ }
break;
case 1: /* Non-option argument. */
--- coreutils-5.1.3/src/unexpand.c.i18n 2004-01-21 22:27:02.000000000 +0000
+++ coreutils-5.1.3/src/unexpand.c 2004-02-16 15:36:40.000000000 +0000
@@ -40,10 +40,27 @@
#include <stdio.h>
#include <getopt.h>
#include <sys/types.h>
+
+/* Get mbstate_t, mbrtowc(), wcwidth(). */
+#if HAVE_WCHAR_H
+# include <wchar.h>
+#endif
+
#include "system.h"
#include "error.h"
#include "posixver.h"
+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
+ installation; work around this configuration error. */
+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
+# define MB_LEN_MAX 16
+#endif
+
+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
+#if HAVE_MBRTOWC && defined mbstate_t
+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "unexpand"
@@ -348,7 +365,8 @@
}
else
{
- ++column;
+ if (!ISCNTRL (c))
+ ++column;
if (convert_entire_line == 0)
convert = 0;
}
@@ -366,6 +384,210 @@
}
}
+#if HAVE_MBRTOWC
+static void
+unexpand_multibyte (void)
+{
+ FILE *fp; /* Input stream. */
+ mbstate_t i_state; /* Current shift state of the input stream. */
+ mbstate_t i_state_bak; /* Back up the I_STATE. */
+ mbstate_t o_state; /* Current shift state of the output stream. */
+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
+ char *bufpos; /* Next read position of BUF. */
+ size_t buflen = 0; /* The length of the byte sequence in buf. */
+ wint_t wc; /* A gotten wide character. */
+ size_t mblength; /* The byte size of a multibyte character
+ which shows as same character as WC. */
+
+ /* Index in `tab_list' of next tabstop: */
+ int tab_index = 0; /* For calculating width of pending tabs. */
+ int print_tab_index = 0; /* For printing as many tabs as possible. */
+ unsigned int column = 0; /* Column on screen of next char. */
+ int next_tab_column; /* Column the next tab stop is on. */
+ int convert = 1; /* If nonzero, perform translations. */
+ unsigned int pending = 0; /* Pending columns of blanks. */
+
+ fp = next_file ((FILE *) NULL);
+ if (fp == NULL)
+ return;
+
+ /* Binary I/O will preserve the original EOL style (DOS/Unix) of files. */
+ SET_BINARY2 (fileno (fp), STDOUT_FILENO);
+
+ memset (&o_state, '\0', sizeof(mbstate_t));
+ memset (&i_state, '\0', sizeof(mbstate_t));
+
+ for (;;)
+ {
+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp))
+ {
+ memmove (buf, bufpos, buflen);
+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp);
+ bufpos = buf;
+ }
+
+ /* Get a wide character. */
+ if (buflen < 1)
+ {
+ mblength = 1;
+ wc = WEOF;
+ }
+ else
+ {
+ i_state_bak = i_state;
+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &i_state);
+ }
+
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ i_state = i_state_bak;
+ wc = L'\0';
+ }
+
+ if (wc == L' ' && convert && column < TAB_STOP_SENTINEL)
+ {
+ ++pending;
+ ++column;
+ }
+ else if (wc == L'\t' && convert)
+ {
+ if (tab_size == 0)
+ {
+ /* Do not let tab_index == first_free_tab;
+ stop when it is 1 less. */
+ while (tab_index < first_free_tab - 1
+ && column >= tab_list[tab_index])
+ tab_index++;
+ next_tab_column = tab_list[tab_index];
+ if (tab_index < first_free_tab - 1)
+ tab_index++;
+ if (column >= next_tab_column)
+ {
+ convert = 0; /* Ran out of tab stops. */
+ goto flush_pend_mb;
+ }
+ }
+ else
+ {
+ next_tab_column = column + tab_size - column % tab_size;
+ }
+ pending += next_tab_column - column;
+ column = next_tab_column;
+ }
+ else
+ {
+flush_pend_mb:
+ /* Flush pending spaces. Print as many tabs as possible,
+ then print the rest as spaces. */
+ if (pending == 1)
+ {
+ putchar (' ');
+ pending = 0;
+ }
+ column -= pending;
+ while (pending > 0)
+ {
+ if (tab_size == 0)
+ {
+ /* Do not let print_tab_index == first_free_tab;
+ stop when it is 1 less. */
+ while (print_tab_index < first_free_tab - 1
+ && column >= tab_list[print_tab_index])
+ print_tab_index++;
+ next_tab_column = tab_list[print_tab_index];
+ if (print_tab_index < first_free_tab - 1)
+ print_tab_index++;
+ }
+ else
+ {
+ next_tab_column =
+ column + tab_size - column % tab_size;
+ }
+ if (next_tab_column - column <= pending)
+ {
+ putchar ('\t');
+ pending -= next_tab_column - column;
+ column = next_tab_column;
+ }
+ else
+ {
+ --print_tab_index;
+ column += pending;
+ while (pending != 0)
+ {
+ putchar (' ');
+ pending--;
+ }
+ }
+ }
+
+ if (wc == WEOF)
+ {
+ fp = next_file (fp);
+ if (fp == NULL)
+ break; /* No more files. */
+ else
+ {
+ memset (&i_state, '\0', sizeof(mbstate_t));
+ SET_BINARY2 (fileno (fp), STDOUT_FILENO);
+ continue;
+ }
+ }
+
+ if (mblength == (size_t)-1 || mblength == (size_t)-2)
+ {
+ if (convert)
+ {
+ ++column;
+ if (convert_entire_line == 0)
+ convert = 0;
+ }
+ mblength = 1;
+ putchar (buf[0]);
+ }
+ else if (mblength == 0)
+ {
+ if (convert && convert_entire_line == 0)
+ convert = 0;
+ mblength = 1;
+ putchar ('\0');
+ }
+ else
+ {
+ if (convert)
+ {
+ if (wc == L'\b')
+ {
+ if (column > 0)
+ --column;
+ }
+ else
+ {
+ int width; /* The width of WC. */
+
+ width = wcwidth (wc);
+ column += (width > 0) ? width : 0;
+ if (convert_entire_line == 0)
+ convert = 0;
+ }
+ }
+
+ if (wc == L'\n')
+ {
+ tab_index = print_tab_index = 0;
+ column = pending = 0;
+ convert = 1;
+ }
+ fwrite (bufpos, sizeof(char), mblength, stdout);
+ }
+ }
+ buflen -= mblength;
+ bufpos += mblength;
+ }
+}
+#endif
+
+
void
usage (int status)
{
@@ -488,7 +710,12 @@
file_list = (optind < argc ? &argv[optind] : stdin_argv);
- unexpand ();
+#if HAVE_MBRTOWC
+ if (MB_CUR_MAX > 1)
+ unexpand_multibyte ();
+ else
+#endif
+ unexpand ();
if (have_read_stdin && fclose (stdin) == EOF)
error (EXIT_FAILURE, errno, "-");
Reference in New Issue View Git Blame Copy Permalink