# HG changeset patch
# User Paul Boddie <paul@boddie.org.uk>
# Date 1487017598 -3600
# Node ID dad54a3a60f9b28a8bab0a0c334715ca0041404a
# Parent  26f221305c300fd1f46462b865a889191d396db8# Parent  aed28d04304df1e594e435b81abd8b242258536a
Merged changes from the attr-strvalue-without-size branch.

diff -r 26f221305c30 -r dad54a3a60f9 generator.py
--- a/generator.py	Mon Feb 13 17:54:29 2017 +0100
+++ b/generator.py	Mon Feb 13 21:26:38 2017 +0100
@@ -505,6 +505,11 @@
                 else:
                     attrs["__key__"] = None
 
+            # Initialise the size, if a string.
+
+            if attrs.has_key("__size__"):
+                attrs["__size__"] = len(data)
+
         # Define Unicode constant encoding details.
 
         if cls == self.unicode_type:
@@ -912,6 +917,12 @@
                                      encode_literal_constant_value(attr)))
                     continue
 
+                # Special internal size member.
+
+                elif attrname == "__size__":
+                    structure.append("{.intvalue=%d}" % attr)
+                    continue
+
                 # Special internal key member.
 
                 elif attrname == "__key__":
diff -r 26f221305c30 -r dad54a3a60f9 lib/__builtins__/str.py
--- a/lib/__builtins__/str.py	Mon Feb 13 17:54:29 2017 +0100
+++ b/lib/__builtins__/str.py	Mon Feb 13 21:26:38 2017 +0100
@@ -22,8 +22,8 @@
 from __builtins__.operator import _negate
 from __builtins__.sequence import hashable, itemaccess
 from __builtins__.types import check_int
-from native import str_add, str_lt, str_gt, str_eq, str_len, str_ord, \
-                   str_nonempty, str_substr
+from native import int_new, str_add, str_lt, str_gt, str_eq, str_ord, \
+                   str_substr
 
 WHITESPACE = (" ", "\f", "\n", "\r", "\t")
 
@@ -39,26 +39,25 @@
         # literals or converted using routines defined for other types, no form
         # of actual initialisation is performed here.
 
+        # Note the __key__ member. This is also initialised statically. Where
+        # a string is the same as an attribute name, the __key__ member contains
+        # attribute position and code details.
+
         # NOTE: Cannot perform "other and other.__data__ or None" since the
         # NOTE: __data__ attribute is not a normal attribute.
 
         if other:
             self.__data__ = other.__data__
+            self.__key__ = other.__key__
+            self.__size__ = other.__size__
         else:
             self.__data__ = None
-
-        # Note the __key__ member. This is also initialised statically. Where
-        # a string is the same as an attribute name, the __key__ member contains
-        # attribute position and code details.
-
-        if other:
-            self.__key__ = other.__key__
-        else:
             self.__key__ = None
+            self.__size__ = None
 
     # Internal methods.
 
-    def _binary_op(self, op, other):
+    def _binary_op(self, op, other, sizes=False):
 
         "Perform 'op' on this object and 'other' if appropriate."
 
@@ -69,10 +68,12 @@
 
         # Otherwise, perform the operation on the operands' data.
 
+        elif sizes:
+            return op(self.__data__, other.__data__, self.__size__, other.__size__)
         else:
             return op(self.__data__, other.__data__)
 
-    def _binary_op_rev(self, op, other):
+    def _binary_op_rev(self, op, other, sizes=False):
 
         "Perform 'op' on 'other' and this object if appropriate."
 
@@ -83,6 +84,8 @@
 
         # Otherwise, perform the operation on the operands' data.
 
+        elif sizes:
+            return op(other.__data__, self.__data__, other.__size__, self.__size__)
         else:
             return op(other.__data__, self.__data__)
 
@@ -154,7 +157,7 @@
 
         "Return the number of bytes in this string."
 
-        return str_len(self.__data__)
+        return int_new(self.__size__)
 
     # General type methods.
 
@@ -162,7 +165,7 @@
 
         "Return whether the string provides any data."
 
-        return str_nonempty(self.__data__)
+        return int_new(self.__size__).__bool__()
 
     def __contains__(self, value):
 
@@ -196,7 +199,7 @@
 
         "Return a string combining this string with 'other'."
 
-        return self._binary_op(str_add, other)
+        return self._binary_op(str_add, other, True)
 
     __add__ = __iadd__
 
@@ -204,7 +207,7 @@
 
         "Return a string combining this string with 'other'."
 
-        return self._binary_op_rev(str_add, other)
+        return self._binary_op_rev(str_add, other, True)
 
     def __mod__(self, other): pass
     def __rmod__(self, other): pass
diff -r 26f221305c30 -r dad54a3a60f9 lib/__builtins__/unicode.py
--- a/lib/__builtins__/unicode.py	Mon Feb 13 17:54:29 2017 +0100
+++ b/lib/__builtins__/unicode.py	Mon Feb 13 21:26:38 2017 +0100
@@ -133,7 +133,7 @@
         "Return the length of this string in characters."
 
         if self.length is None:
-            self.length = unicode_len(self.__data__)
+            self.length = unicode_len(self.__data__, self.__size__)
 
         return self.length
 
@@ -142,7 +142,7 @@
         "Return the value of the string, if only a single character."
 
         if self.__len__() == 1:
-            return unicode_ord(self.__data__)
+            return unicode_ord(self.__data__, self.__size__)
         else:
             raise ValueError, self
 
@@ -204,7 +204,7 @@
         "Return the item at the normalised (positive) 'index'."
     
         self._check_index(index)
-        return utf8string(unicode_substr(self.__data__, index, index + 1, 1), self.encoding)
+        return utf8string(unicode_substr(self.__data__, self.__size__, index, index + 1, 1), self.encoding)
 
     def __get_multiple_items__(self, start, end, step):
 
diff -r 26f221305c30 -r dad54a3a60f9 lib/native/__init__.py
--- a/lib/native/__init__.py	Mon Feb 13 17:54:29 2017 +0100
+++ b/lib/native/__init__.py	Mon Feb 13 21:26:38 2017 +0100
@@ -23,7 +23,8 @@
 
 from native.identity import is_, is_not
 
-from native.int import int_add, int_div, int_mod, int_mul, int_neg, int_pow, \
+from native.int import int_new, \
+                       int_add, int_div, int_mod, int_mul, int_neg, int_pow, \
                        int_sub, int_and, int_not, int_or, int_xor, int_lt, \
                        int_gt, int_eq, int_ne, int_str
 
@@ -42,8 +43,8 @@
 
 from native.program import get_using
 
-from native.str import str_add, str_chr, str_eq, str_gt, str_lt, str_len, \
-                       str_nonempty, str_ord, str_substr
+from native.str import str_add, str_chr, str_eq, str_gt, str_lt, \
+                       str_ord, str_substr
 
 from native.system import exit, get_argv, get_path
 
diff -r 26f221305c30 -r dad54a3a60f9 lib/native/int.py
--- a/lib/native/int.py	Mon Feb 13 17:54:29 2017 +0100
+++ b/lib/native/int.py	Mon Feb 13 21:26:38 2017 +0100
@@ -8,7 +8,7 @@
 non-core exceptions used by the native functions because they need to be
 identified as being needed by the program.
 
-Copyright (C) 2011, 2015, 2016 Paul Boddie <paul@boddie.org.uk>
+Copyright (C) 2011, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk>
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -24,6 +24,8 @@
 this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+def int_new(data): pass
+
 def int_add(self, other): pass
 def int_div(self, other): pass
 def int_mod(self, other): pass
diff -r 26f221305c30 -r dad54a3a60f9 lib/native/str.py
--- a/lib/native/str.py	Mon Feb 13 17:54:29 2017 +0100
+++ b/lib/native/str.py	Mon Feb 13 21:26:38 2017 +0100
@@ -26,13 +26,11 @@
 
 # String operations.
 
-def str_add(data, other_data): pass
+def str_add(data, other_data, size, other_size): pass
 def str_chr(data): pass
 def str_eq(data, other_data): pass
 def str_gt(data, other_data): pass
 def str_lt(data, other_data): pass
-def str_len(data): pass
-def str_nonempty(data): pass
 def str_ord(data): pass
 def str_substr(data, start, end, step): pass
 
diff -r 26f221305c30 -r dad54a3a60f9 lib/native/unicode.py
--- a/lib/native/unicode.py	Mon Feb 13 17:54:29 2017 +0100
+++ b/lib/native/unicode.py	Mon Feb 13 21:26:38 2017 +0100
@@ -26,8 +26,8 @@
 
 # Unicode string operations.
 
-def unicode_len(data): pass
-def unicode_ord(data): pass
-def unicode_substr(data, start, end, step): pass
+def unicode_len(data, size): pass
+def unicode_ord(data, size): pass
+def unicode_substr(data, size, start, end, step): pass
 
 # vim: tabstop=4 expandtab shiftwidth=4
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/buffer.c
--- a/templates/native/buffer.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/buffer.c	Mon Feb 13 21:26:38 2017 +0100
@@ -37,7 +37,7 @@
 
     /* Calculate the size of the string. */
     for (i = 0; i < data->size; i++)
-        size += strlen(__load_via_object(data->attrs[i].value, __pos___data__).strvalue);
+        size += __load_via_object(data->attrs[i].value, __pos___size__).intvalue;
 
     /* Reserve space for a new string. */
     s = (char *) __ALLOCATE(size + 1, sizeof(char));
@@ -46,13 +46,13 @@
     for (i = 0, j = 0; i < data->size; i++)
     {
         o = __load_via_object(data->attrs[i].value, __pos___data__);
-        n = strlen(o.strvalue);
+        n = __load_via_object(data->attrs[i].value, __pos___size__).intvalue;
         memcpy(s + j, o.strvalue, n); /* does not null terminate but final byte should be zero */
         j += n;
     }
 
     /* Return a new string. */
-    return __new_str(s);
+    return __new_str(s, size);
 }
 
 /* Module initialisation. */
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/common.c
--- a/templates/native/common.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/common.c	Mon Feb 13 21:26:38 2017 +0100
@@ -34,11 +34,12 @@
     return attr;
 }
 
-__attr __new_str(char *s)
+__attr __new_str(char *s, int size)
 {
-    /* Create a new string and mutate the __data__ and __key__ attributes. */
+    /* Create a new string and mutate the __data__, __size__ and __key__ attributes. */
     __attr attr = __NEWINSTANCE(__builtins___str_string);
     attr.value->attrs[__pos___data__].strvalue = s;
+    attr.value->attrs[__pos___size__].intvalue = size;
     attr.value->attrs[__pos___key__] = __NULL;
     return attr;
 }
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/common.h
--- a/templates/native/common.h	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/common.h	Mon Feb 13 21:26:38 2017 +0100
@@ -24,7 +24,7 @@
 /* Utility functions. */
 
 __attr __new_int(int i);
-__attr __new_str(char *s);
+__attr __new_str(char *s, int size);
 __attr __new_list(__fragment *f);
 __fragment *__fragment_append(__fragment *data, __attr * const value);
 
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/iconv.c
--- a/templates/native/iconv.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/iconv.c	Mon Feb 13 21:26:38 2017 +0100
@@ -99,9 +99,9 @@
         /* Incomplete sequence: raise the string in an OSError instead. */
 
         if (errno == EINVAL)
-            __raise_incomplete_sequence_error(__new_int(errno), __new_str(resultbuf));
+            __raise_incomplete_sequence_error(__new_int(errno), __new_str(resultbuf, outbytestotal));
 
-        return __new_str(resultbuf);
+        return __new_str(resultbuf, outbytestotal);
     }
 
     /* Invalid sequence. */
@@ -110,7 +110,7 @@
     {
         resultbuf = __ALLOCATE(inbytesleft + 1, sizeof(char));
         memcpy(resultbuf, inbuf, inbytesleft);
-        __raise_invalid_sequence_error(__new_int(errno), __new_str(resultbuf));
+        __raise_invalid_sequence_error(__new_int(errno), __new_str(resultbuf, inbytesleft));
     }
 
     /* General failure. */
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/int.c
--- a/templates/native/int.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/int.c	Mon Feb 13 21:26:38 2017 +0100
@@ -18,8 +18,9 @@
 
 #include <limits.h> /* INT_MAX, INT_MIN */
 #include <math.h>   /* ceil, log10, pow */
-#include <stdio.h>  /* fdopen, snprintf, strlen */
+#include <stdio.h>  /* fdopen, snprintf */
 #include <errno.h>  /* errno */
+#include <string.h> /* strlen */
 #include "native/common.h"
 #include "types.h"
 #include "exceptions.h"
@@ -31,6 +32,13 @@
 
 /* Integer operations. */
 
+__attr __fn_native_int_int_new(__attr __args[])
+{
+    __attr * const _data = &__args[1];
+
+    return __new_int(_data->intvalue);
+}
+
 __attr __fn_native_int_int_add(__attr __args[])
 {
     __attr * const _data = &__args[1];
@@ -268,7 +276,7 @@
     snprintf(s, n, "%d", i);
 
     /* Return a new string. */
-    return __new_str(s);
+    return __new_str(s, strlen(s));
 }
 
 /* Module initialisation. */
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/int.h
--- a/templates/native/int.h	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/int.h	Mon Feb 13 21:26:38 2017 +0100
@@ -1,6 +1,6 @@
 /* Native functions for integer operations.
 
-Copyright (C) 2016 Paul Boddie <paul@boddie.org.uk>
+Copyright (C) 2016, 2017 Paul Boddie <paul@boddie.org.uk>
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -23,6 +23,8 @@
 
 /* Integer operations. */
 
+__attr __fn_native_int_int_new(__attr __args[]);
+
 __attr __fn_native_int_int_add(__attr __args[]);
 __attr __fn_native_int_int_div(__attr __args[]);
 __attr __fn_native_int_int_mod(__attr __args[]);
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/io.c
--- a/templates/native/io.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/io.c	Mon Feb 13 21:26:38 2017 +0100
@@ -148,7 +148,7 @@
 
     s = __ALLOCATE(have_read + 1, sizeof(char));
     memcpy(s, (char *) buf, have_read); /* does not null terminate but final byte should be zero */
-    return __new_str(s);
+    return __new_str(s, have_read);
 }
 
 __attr __fn_native_io_fwrite(__attr __args[])
@@ -158,9 +158,9 @@
     /* fp interpreted as FILE reference */
     FILE *f = (FILE *) fp->datavalue;
     /* str.__data__ interpreted as string */
-    __attr sa = __load_via_object(str->value, __pos___data__);
-    char *s = sa.strvalue;
-    size_t to_write = strlen(sa.strvalue);
+    char *s = __load_via_object(str->value, __pos___data__).strvalue;
+    /* str.__size__ interpreted as int */
+    int to_write = __load_via_object(str->value, __pos___size__).intvalue;
     size_t have_written = fwrite(s, sizeof(char), to_write, f);
     int error;
 
@@ -210,7 +210,7 @@
 
     s = __ALLOCATE(have_read + 1, 1);
     memcpy(s, (char *) buf, have_read); /* does not null terminate but final byte should be zero */
-    return __new_str(s);
+    return __new_str(s, have_read);
 }
 
 __attr __fn_native_io_write(__attr __args[])
@@ -220,12 +220,13 @@
     /* fd.__data__ interpreted as int */
     int i = __load_via_object(fd->value, __pos___data__).intvalue;
     /* str.__data__ interpreted as string */
-    __attr sa = __load_via_object(str->value, __pos___data__);
-    char *s = sa.strvalue;
+    char *s = __load_via_object(str->value, __pos___data__).strvalue;
+    /* str.__size__ interpreted as int */
+    int size = __load_via_object(str->value, __pos___size__).intvalue;
     ssize_t have_written;
 
     errno = 0;
-    have_written = write(i, s, sizeof(char) * strlen(sa.strvalue));
+    have_written = write(i, s, sizeof(char) * size);
 
     if (have_written == -1)
         __raise_io_error(__new_int(errno));
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/locale.c
--- a/templates/native/locale.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/locale.c	Mon Feb 13 21:26:38 2017 +0100
@@ -46,7 +46,7 @@
     out = __ALLOCATE(length + 1, sizeof(char));
     strncpy(out, result, length);
 
-    return __new_str(result);
+    return __new_str(result, length);
 }
 
 __attr __fn_native_locale_setlocale(__attr __args[])
@@ -69,7 +69,7 @@
     out = __ALLOCATE(length + 1, sizeof(char));
     strncpy(out, result, length);
 
-    return __new_str(result);
+    return __new_str(result, length);
 }
 
 /* Module initialisation. */
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/str.c
--- a/templates/native/str.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/str.c	Mon Feb 13 21:26:38 2017 +0100
@@ -32,10 +32,12 @@
 {
     __attr * const _data = &__args[1];
     __attr * const other = &__args[2];
+    __attr * const _size = &__args[3];
+    __attr * const othersize = &__args[4];
     /* _data, other interpreted as string */
     char *s = _data->strvalue;
     char *o = other->strvalue;
-    size_t ss = strlen(_data->strvalue), os = strlen(other->strvalue);
+    int ss = _size->intvalue, os = othersize->intvalue;
     int n = ss + os;
     char *r = (char *) __ALLOCATE(n + 1, sizeof(char));
 
@@ -43,7 +45,7 @@
     memcpy(r + ss, o, os);
 
     /* Return a new string. */
-    return __new_str(r);
+    return __new_str(r, n);
 }
 
 __attr __fn_native_str_str_chr(__attr __args[])
@@ -54,7 +56,7 @@
     char *s = (char *) __ALLOCATE(2, sizeof(char));
 
     s[0] = (char) n;
-    return __new_str(s);
+    return __new_str(s, 1);
 }
 
 __attr __fn_native_str_str_lt(__attr __args[])
@@ -93,21 +95,6 @@
     return strcmp(s, o) == 0 ? __builtins___boolean_True : __builtins___boolean_False;
 }
 
-__attr __fn_native_str_str_len(__attr __args[])
-{
-    __attr * const _data = &__args[1];
-
-    /* Return the new integer. */
-    return __new_int(strlen(_data->strvalue));
-}
-
-__attr __fn_native_str_str_nonempty(__attr __args[])
-{
-    __attr * const _data = &__args[1];
-
-    return _data->strvalue[0] ? __builtins___boolean_True : __builtins___boolean_False;
-}
-
 __attr __fn_native_str_str_ord(__attr __args[])
 {
     __attr * const _data = &__args[1];
@@ -147,7 +134,7 @@
         for (from = istart, to = 0; from > iend; from += istep, to++)
             sub[to] = s[from];
 
-    return __new_str(sub);
+    return __new_str(sub, resultsize);
 }
 
 /* Module initialisation. */
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/str.h
--- a/templates/native/str.h	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/str.h	Mon Feb 13 21:26:38 2017 +0100
@@ -26,8 +26,6 @@
 __attr __fn_native_str_str_lt(__attr __args[]);
 __attr __fn_native_str_str_gt(__attr __args[]);
 __attr __fn_native_str_str_eq(__attr __args[]);
-__attr __fn_native_str_str_len(__attr __args[]);
-__attr __fn_native_str_str_nonempty(__attr __args[]);
 __attr __fn_native_str_str_ord(__attr __args[]);
 __attr __fn_native_str_str_substr(__attr __args[]);
 
diff -r 26f221305c30 -r dad54a3a60f9 templates/native/unicode.c
--- a/templates/native/unicode.c	Mon Feb 13 17:54:29 2017 +0100
+++ b/templates/native/unicode.c	Mon Feb 13 21:26:38 2017 +0100
@@ -72,11 +72,14 @@
 __attr __fn_native_unicode_unicode_len(__attr __args[])
 {
     __attr * const _data = &__args[1];
+    __attr * const _size = &__args[2];
     /* _data interpreted as string */
     char *s = _data->strvalue;
+    /* _size interpreted as int */
+    int size = _size->intvalue;
     unsigned int i, c = 0;
 
-    for (i = 0; s[i] != 0; i++)
+    for (i = 0; i < size; i++)
         if (boundary(s[i]))
             c++;
 
@@ -87,11 +90,14 @@
 __attr __fn_native_unicode_unicode_ord(__attr __args[])
 {
     __attr * const _data = &__args[1];
+    __attr * const _size = &__args[2];
     /* _data interpreted as string */
     char *s = _data->strvalue;
+    /* _size interpreted as int */
+    int size = _size->intvalue;
     unsigned int i, c = 0, v;
 
-    for (i = 0; s[i] != 0; i++)
+    for (i = 0; i < size; i++)
     {
         /* Evaluate the current character as a boundary. */
 
@@ -120,11 +126,14 @@
 __attr __fn_native_unicode_unicode_substr(__attr __args[])
 {
     __attr * const _data = &__args[1];
-    __attr * const start = &__args[2];
-    __attr * const end = &__args[3];
-    __attr * const step = &__args[4];
+    __attr * const _size = &__args[2];
+    __attr * const start = &__args[3];
+    __attr * const end = &__args[4];
+    __attr * const step = &__args[5];
     /* _data interpreted as string */
     char *s = _data->strvalue, *sub;
+    /* _size interpreted as int */
+    int ss = _size->intvalue;
     /* start.__data__ interpreted as int */
     int istart = __load_via_object(start->value, __pos___data__).intvalue;
     /* end.__data__ interpreted as int */
@@ -137,7 +146,7 @@
     unsigned int indexes[nchar];
 
     unsigned int c, d, i, to, from, lastbyte = 0;
-    size_t resultsize = 0, ss = strlen(_data->strvalue);
+    int resultsize = 0;
 
     /* Find the indexes of the characters. */
     if (istep > 0)
@@ -190,7 +199,7 @@
         } while (!boundary(s[from]));
     }
 
-    return __new_str(sub);
+    return __new_str(sub, resultsize);
 }
 
 /* Module initialisation. */