coreutils/coreutils-i18n-cut.patch

--- coreutils-8.24/src/cut.c	2015-06-26 19:05:22.000000000 +0200
+++ cut.c	2016-01-15 10:15:04.863804121 +0100
@@ -28,6 +28,11 @@
 #include <assert.h>
 #include <getopt.h>
 #include <sys/types.h>
+
+#include <mbfile.h>
+#include <mbiter.h>
+#include <string.h>
+
 #include "system.h"
 
 #include "error.h"
@@ -90,25 +95,16 @@ add_range_pair (size_t lo, size_t hi)
    CURRENT_RP.HI then we make CURRENT_RP to point to the next range pair. */
 static struct field_range_pair *current_rp;
 
-/* This buffer is used to support the semantics of the -s option
-   (or lack of same) when the specified field list includes (does
-   not include) the first field.  In both of those cases, the entire
-   first field must be read into this buffer to determine whether it
-   is followed by a delimiter or a newline before any of it may be
-   output.  Otherwise, cut_fields can do the job without using this
-   buffer.  */
-static char *field_1_buffer;
-
-/* The number of bytes allocated for FIELD_1_BUFFER.  */
-static size_t field_1_bufsize;
-
 enum operating_mode
   {
     undefined_mode,
 
-    /* Output characters that are in the given bytes. */
+    /* Output the given bytes. */
     byte_mode,
 
+    /* Output characters that are in the given positions . */
+    char_mode,
+
     /* Output the given delimiter-separated fields. */
     field_mode
   };
@@ -120,12 +116,16 @@ static enum operating_mode operating_mod
    with field mode.  */
 static bool suppress_non_delimited;
 
+/* Unless true, we do not recognize multibyte characters in byte-splitting
+   mode.  */
+static bool no_break_mb_chars;
+
 /* If true, print all bytes, characters, or fields _except_
    those that were specified.  */
 static bool complement;
 
 /* The delimiter character for field mode. */
-static unsigned char delim;
+static mbf_char_t delim;
 
 /* The delimiter for each line/record. */
 static unsigned char line_delim = '\n';
@@ -135,7 +135,7 @@ static size_t output_delimiter_length;
 
 /* The output field separator string.  Defaults to the 1-character
    string consisting of the input delimiter.  */
-static char *output_delimiter_string;
+static char const *output_delimiter_string;
 
 /* True if we have ever read standard input. */
 static bool have_read_stdin;
@@ -189,7 +189,7 @@ Print selected parts of lines from each
   -f, --fields=LIST       select only these fields;  also print any line\n\
                             that contains no delimiter character, unless\n\
                             the -s option is specified\n\
-  -n                      (ignored)\n\
+  -n                      with -b, don't split multibyte characters\n\
 "), stdout);
       fputs (_("\
       --complement        complement the set of selected bytes, characters\n\
@@ -435,6 +435,12 @@ next_item (size_t *item_idx)
     current_rp++;
 }
 
+static inline void
+next_item_n (size_t *item_idx, size_t n)
+{
+  while (n-- > 0)
+    next_item (item_idx);
+}
 /* Return nonzero if the K'th field or byte is printable. */
 
 static inline bool
@@ -443,6 +449,15 @@ print_kth (size_t k)
   return current_rp->lo <= k;
 }
 
+/* The lo and hi params should be used for the current characters byte position
+ * and byte size, respectively. */
+static inline bool
+rp_intersect (size_t lo, size_t hi)
+{
+  return ((current_rp->lo <= lo && current_rp->hi >= lo)
+	  || (current_rp->lo <= hi && current_rp->hi >= hi));
+}
+
 /* Return nonzero if K'th byte is the beginning of a range. */
 
 static inline bool
@@ -505,23 +520,215 @@ cut_bytes (FILE *stream)
 }
 
 /* Read from stream STREAM, printing to standard output any selected fields.  */
+extern ssize_t
+mb_getndelim2 (mbf_char_t **lineptr, size_t *linesize, size_t nmax,
+	       mbf_char_t delim1, mbf_char_t delim2, mb_file_t *stream)
+{
+/* The maximum value that getndelim2 can return without suffering from
+   overflow problems, either internally (because of pointer
+   subtraction overflow) or due to the API (because of ssize_t).  */
+#define GETNDELIM2_MAXIMUM (PTRDIFF_MAX < SSIZE_MAX ? PTRDIFF_MAX : SSIZE_MAX)
+
+/* Try to add at least this many bytes when extending the buffer.
+   MIN_CHUNK must be no greater than GETNDELIM2_MAXIMUM.  */
+#define MIN_CHUNK 64
+  size_t nchars_avail;          /* Allocated but unused chars in *LINEPTR.  */
+  mbf_char_t *read_pos;               /* Where we're reading into *LINEPTR. */
+  ssize_t chars_stored = -1;
+  mbf_char_t *ptr = *lineptr;
+  size_t size = *linesize;
+  bool found_delimiter;
+
+  if (!ptr)
+    {
+      size = nmax < MIN_CHUNK ? nmax : MIN_CHUNK;
+      ptr = malloc (size * sizeof (mbf_char_t));
+      if (!ptr)
+        return -1;
+    }
+
+  if (size < 0)
+    goto done;
+
+  nchars_avail = size;
+  read_pos = ptr;
+
+  if (nchars_avail == 0 && nmax <= size)
+    goto done;
+
+  /* Normalize delimiters, since memchr2 doesn't handle EOF.  */
+  if (mb_iseof (delim1))
+    mb_copy (&delim1, &delim2);
+  else if (mb_iseof (delim2))
+    mb_copy (&delim2, &delim1);
+
+  flockfile (stream);
+
+  found_delimiter = false;
+  do
+    {
+      /* Here always ptr + size == read_pos + nchars_avail.
+         Also nchars_avail > 0 || size < nmax.  */
+
+      mbf_char_t c IF_LINT (= 0);
+        {
+          mbf_getc (c, *stream);
+          if (mb_iseof (c))
+            {
+              /* Return partial line, if any.  */
+              if (read_pos == ptr)
+                goto unlock_done;
+              else
+                break;
+            }
+          if (mb_equal (c, delim1) || mb_equal (c, delim2))
+            found_delimiter = true;
+        }
+
+      /* We always want at least one byte left in the buffer, since we
+         always (unless we get an error while reading the first byte)
+         NUL-terminate the line buffer.  */
+
+      if (!nchars_avail)
+        {
+          /* Grow size proportionally, not linearly, to avoid O(n^2)
+             running time.  */
+          size_t newsize = size < MIN_CHUNK ? size + MIN_CHUNK : 2 * size;
+          mbf_char_t *newptr;
+
+          /* Respect nmax.  This handles possible integer overflow.  */
+          if (! (size < newsize && newsize <= nmax))
+            newsize = nmax;
+
+          if (GETNDELIM2_MAXIMUM < newsize)
+            {
+              size_t newsizemax = GETNDELIM2_MAXIMUM + 1;
+              if (size == newsizemax)
+                goto unlock_done;
+              newsize = newsizemax;
+            }
+          nchars_avail = newsize - (read_pos - ptr);
+          newptr = realloc (ptr, newsize * sizeof (mbf_char_t));
+          if (!newptr)
+            goto unlock_done;
+          ptr = newptr;
+          size = newsize;
+          read_pos = size - nchars_avail + ptr;
+        }
+
+      /* Here, if size < nmax, nchars_avail >= buffer_len + 1.
+         If size == nmax, nchars_avail > 0.  */
+
+      if (1 < nchars_avail--)
+        {
+          mb_copy(read_pos++, &c);
+        }
+
+    }
+  while (!found_delimiter);
+
+  chars_stored = (read_pos - ptr);
+
+ unlock_done:
+  funlockfile (stream);
+
+ done:
+  *lineptr = ptr;
+  *linesize = size;
+  return chars_stored;
+}
+
+static void
+cut_chars (FILE *stream)
+{
+  size_t char_idx;	/* Number of chars in the line so far. */
+  bool print_delimiter;
+  mbf_char_t c;
+  mb_file_t mbf;
+
+  print_delimiter = false;
+  char_idx = 0;
+  current_rp = frp;
+
+  mbf_init (mbf, stream);
+  while (true)
+    {
+      mbf_getc (c, mbf);
+
+      if (mb_iseq (c, line_delim))
+	{
+	  putc (line_delim, stdout);
+	  char_idx = 0;
+	  print_delimiter = false;
+          current_rp = frp;
+	}
+      else if (mb_iseof (c))
+	{
+	  if (char_idx > 0)
+	    putc (line_delim, stdout);
+	  break;
+	}
+      else
+	{
+	  /* Forward by one byte.  */
+	  next_item (&char_idx);
+
+	  /* Check if the current characters byte range is within
+	   * the argument list.  */
+	  if (rp_intersect (char_idx, char_idx + mb_len (c) - 1))
+	    {
+	      if (output_delimiter_specified)
+		{
+		  if (print_delimiter && is_range_start_index (char_idx))
+		    {
+		      fwrite (output_delimiter_string, sizeof (char),
+			      output_delimiter_length, stdout);
+		    }
+		  print_delimiter = true;
+		}
+	      mb_putc (c, stdout);
+	    }
+
+	  /* Byte mode with multibyte characters uncut (-b -n).  */
+	  if (no_break_mb_chars)
+	    /* Forward by an additional byte_length (c) - 1.  */
+	    next_item_n (&char_idx, mb_len (c) - 1);
+       }
+    }
+}
 
 static void
 cut_fields (FILE *stream)
 {
-  int c;
+
+  /* This buffer is used to support the semantics of the -s option
+     (or lack of same) when the specified field list includes (does
+     not include) the first field.  In both of those cases, the entire
+     first field must be read into this buffer to determine whether it
+     is followed by a delimiter or a newline before any of it may be
+     output.  Otherwise, cut_fields can do the job without using this
+     buffer.  */
+  mbf_char_t *field_1_buffer = 0;
+  /* The number of bytes allocated for FIELD_1_BUFFER.  */
+  size_t field_1_bufsize;
+
+
+  mbf_char_t c, d;
+  mb_file_t mbf;
   size_t field_idx = 1;
   bool found_any_selected_field = false;
   bool buffer_first_field;
 
   current_rp = frp;
 
-  c = getc (stream);
-  if (c == EOF)
+  mbf_init (mbf, stream);
+  mbf_getc (c, mbf);
+  if (mb_iseof (c))
     return;
 
-  ungetc (c, stream);
-  c = 0;
+  mbf_ungetc (c, mbf);
+  mb_setascii (&c, 0);
+  mb_copy (&d, &delim);
 
   /* To support the semantics of the -s flag, we may have to buffer
      all of the first field to determine whether it is 'delimited.'
@@ -536,10 +744,14 @@ cut_fields (FILE *stream)
       if (field_idx == 1 && buffer_first_field)
         {
           ssize_t len;
-          size_t n_bytes;
+          size_t n_chars;
+	  mbf_char_t nl;
+	  mb_setascii (&nl, line_delim);
+
+          len = mb_getndelim2 (&field_1_buffer, &field_1_bufsize,
+			       GETNLINE_NO_LIMIT, d, nl, &mbf);
+
 
-          len = getndelim2 (&field_1_buffer, &field_1_bufsize, 0,
-                            GETNLINE_NO_LIMIT, delim, line_delim, stream);
           if (len < 0)
             {
               free (field_1_buffer);
@@ -549,15 +761,15 @@ cut_fields (FILE *stream)
               xalloc_die ();
             }
 
-          n_bytes = len;
-          assert (n_bytes != 0);
+          n_chars = len;
+          //assert (n_chars != 0);
 
-          c = 0;
+	  mb_setascii (&c, 0);
 
           /* If the first field extends to the end of line (it is not
              delimited) and we are printing all non-delimited lines,
              print this one.  */
-          if (to_uchar (field_1_buffer[n_bytes - 1]) != delim)
+          if (!mb_equal (field_1_buffer[n_chars - 1], d))
             {
               if (suppress_non_delimited)
                 {
@@ -565,26 +777,30 @@ cut_fields (FILE *stream)
                 }
               else
                 {
-                  fwrite (field_1_buffer, sizeof (char), n_bytes, stdout);
+		  for (int i = 0; i < n_chars; ++i)
+		    mb_putc (field_1_buffer[i], stdout);
+
                   /* Make sure the output line is newline terminated.  */
-                  if (field_1_buffer[n_bytes - 1] != line_delim)
+                  if (!mb_iseq (field_1_buffer[n_chars - 1], line_delim))
                     putchar (line_delim);
-                  c = line_delim;
+                  mb_setascii (&c, line_delim);
                 }
               continue;
             }
           if (print_kth (1))
             {
               /* Print the field, but not the trailing delimiter.  */
-              fwrite (field_1_buffer, sizeof (char), n_bytes - 1, stdout);
+	      for (int i = 0; i < n_chars - 1; ++i)
+		mb_putc (field_1_buffer[i], stdout);
 
               /* With -d$'\n' don't treat the last '\n' as a delimiter.  */
-              if (delim == line_delim)
+              if (mb_iseq (d, line_delim))
                 {
-                  int last_c = getc (stream);
-                  if (last_c != EOF)
+                  mbf_char_t last_c;
+		  mbf_getc (last_c, mbf);
+                  if (!mb_iseof (last_c))
                     {
-                      ungetc (last_c, stream);
+                      mbf_ungetc (last_c, mbf);
                       found_any_selected_field = true;
                     }
                 }
@@ -594,7 +810,8 @@ cut_fields (FILE *stream)
           next_item (&field_idx);
         }
 
-      int prev_c = c;
+      mbf_char_t prev_c;
+      mb_copy (&prev_c, &c);
 
       if (print_kth (field_idx))
         {
@@ -605,42 +822,46 @@ cut_fields (FILE *stream)
             }
           found_any_selected_field = true;
 
-          while ((c = getc (stream)) != delim && c != line_delim && c != EOF)
+	  mbf_getc (c, mbf);
+          while (!mb_equal (c, d) && !mb_iseq (c, line_delim) && !mb_iseof (c))
             {
-              putchar (c);
-              prev_c = c;
+              mb_putc (c, stdout);
+	      mb_copy (&prev_c, &c);
+	      mbf_getc (c, mbf);
             }
         }
       else
         {
-          while ((c = getc (stream)) != delim && c != line_delim && c != EOF)
+	  mbf_getc (c, mbf);
+          while (!mb_equal (c, d) && !mb_iseq (c, line_delim) && !mb_iseof (c))
             {
-              prev_c = c;
+	      mb_copy (&prev_c, &c);
+	      mbf_getc (c, mbf);
             }
         }
 
       /* With -d$'\n' don't treat the last '\n' as a delimiter.  */
-      if (delim == line_delim && c == delim)
+      if (mb_iseq (d, line_delim) && mb_equal (c, d))
         {
-          int last_c = getc (stream);
-          if (last_c != EOF)
-            ungetc (last_c, stream);
+	  mbf_char_t last_c;
+	  mbf_getc (last_c, mbf);
+	  if (!mb_iseof (last_c))
+	    mbf_ungetc (last_c, mbf);
           else
-            c = last_c;
+	    mb_copy (&c, &last_c);
         }
 
-      if (c == delim)
+      if (mb_equal (c, d))
         next_item (&field_idx);
-      else if (c == line_delim || c == EOF)
+      else if (mb_iseq (c, line_delim) || mb_iseof (c))
         {
           if (found_any_selected_field
               || !(suppress_non_delimited && field_idx == 1))
             {
-              if (c == line_delim || prev_c != line_delim
-                  || delim == line_delim)
+              if (mb_iseq (c, line_delim) || !mb_iseq (prev_c, line_delim) || mb_iseq (d, line_delim))
                 putchar (line_delim);
             }
-          if (c == EOF)
+          if (mb_iseof (c))
             break;
           field_idx = 1;
           current_rp = frp;
@@ -652,7 +874,14 @@ static void
 cut_stream (FILE *stream)
 {
   if (operating_mode == byte_mode)
-    cut_bytes (stream);
+    {
+      if (no_break_mb_chars)
+	cut_chars (stream);
+      else
+	cut_bytes (stream);
+    }
+  else if (operating_mode == char_mode)
+    cut_chars (stream);
   else
     cut_fields (stream);
 }
@@ -706,6 +935,7 @@ main (int argc, char **argv)
   bool ok;
   bool delim_specified = false;
   char *spec_list_string IF_LINT ( = NULL);
+  mbi_iterator_t iter;
 
   initialize_main (&argc, &argv);
   set_program_name (argv[0]);
@@ -719,8 +949,10 @@ main (int argc, char **argv)
 
   /* By default, all non-delimited lines are printed.  */
   suppress_non_delimited = false;
+  /* Default behaviour for -b, unless -n is also specified.  */
+  no_break_mb_chars = false;
 
-  delim = '\0';
+  mb_setascii (&delim, '\0');
   have_read_stdin = false;
 
   while ((optc = getopt_long (argc, argv, "b:c:d:f:nsz", longopts, NULL)) != -1)
@@ -728,7 +960,6 @@ main (int argc, char **argv)
       switch (optc)
         {
         case 'b':
-        case 'c':
           /* Build the byte list. */
           if (operating_mode != undefined_mode)
             FATAL_ERROR (_("only one type of list may be specified"));
@@ -736,6 +967,14 @@ main (int argc, char **argv)
           spec_list_string = optarg;
           break;
 
+        case 'c':
+          /* Build the char list. */
+          if (operating_mode != undefined_mode)
+            FATAL_ERROR (_("only one type of list may be specified"));
+          operating_mode = char_mode;
+          spec_list_string = optarg;
+          break;
+
         case 'f':
           /* Build the field list. */
           if (operating_mode != undefined_mode)
@@ -747,9 +986,17 @@ main (int argc, char **argv)
         case 'd':
           /* New delimiter. */
           /* Interpret -d '' to mean 'use the NUL byte as the delimiter.'  */
-          if (optarg[0] != '\0' && optarg[1] != '\0')
+	  mbi_init (iter, optarg, strlen (optarg));
+	  if  (!mbi_avail (iter))
+	    mb_setascii (&delim, '\0');
+	  else
+           {
+             mb_copy (&delim, &mbi_cur (iter));
+
+             mbi_advance (iter);
+             if (mbi_avail (iter))
             FATAL_ERROR (_("the delimiter must be a single character"));
+           }
-          delim = optarg[0];
           delim_specified = true;
           break;
 
@@ -763,6 +1008,7 @@ main (int argc, char **argv)
           break;
 
         case 'n':
+	  no_break_mb_chars = true;
           break;
 
         case 's':
@@ -802,15 +1048,12 @@ main (int argc, char **argv)
               | (complement ? SETFLD_COMPLEMENT : 0) );
 
   if (!delim_specified)
-    delim = '\t';
+    mb_setascii (&delim, '\t');
 
   if (output_delimiter_string == NULL)
     {
-      static char dummy[2];
-      dummy[0] = delim;
-      dummy[1] = '\0';
-      output_delimiter_string = dummy;
-      output_delimiter_length = 1;
+      output_delimiter_string = mb_ptr (delim);
+      output_delimiter_length = mb_len (delim);
     }
 
   if (optind == argc)