#############################################################################
#                                                                           #
# file : utf8.icn                                                           #
#                                                                           #
# Author : Bruce Rennie                                                     #
# Date   : 7-August-2013                                                    #
# Update : 6-April-2015                                                     #
# Update : 5-July-2015                                                      #
# Update : 4-September-2018 through to 24-October-2018                      #
# Update : 7-September-2020                                                 #
# Update : 16-January-2021                                                 #
#                                                                           #
# Purpose:  To provide a set of procedures for processing UTF-8 streams     #
#           This set of routines is incomplete and will not handle          #
#           other forms of Unicode encoding.                                #
#           This has now been updated to provide a class that can use       #
#           the string scanning to locate and extract UTF-8 strings         #
#           from an UTF-8 source string.                                    #
#                                                                           #
# Description:                                                              #
# UTF-8 character encoding/decoding                                         #
# The following has been taken from the following URL:                      #
#                                                                           #
#         http://en.wikipedia.org/wiki/UTF-8                                #
#                                                                           #
# The design of UTF-8 can be seen in this table of the scheme as            #
# originally proposed by Dave Prosser and subsequently modified by Ken      #
# Thompson (the x's are replaced by the bits of the code point):            #
#                                                                           #
# Bits of     First       Last        Bytes in    Byte 1      Byte 2        #
# code point  code point  code point  sequence                              #
#                                                                           #
#     7       U+0000      U+007F          1       0xxxxxxx                  #
#     11      U+0080      U+07FF          2       110xxxxx    10xxxxxx      #
#     16      U+0800      U+FFFF          3       1110xxxx    10xxxxxx      #
#     21      U+10000     U+1FFFFF        4       11110xxx    10xxxxxx      #
#     26      U+200000    U+3FFFFFF       5       111110xx    10xxxxxx      #
#     31      U+4000000   U+7FFFFFFF      6       1111110x    10xxxxxx      #
#                                                                           #
#             Byte 3      Byte 4      Byte 5      Byte 6                    #
#                                                                           #
#     7                                                                     #
#     11                                                                    #
#     16      10xxxxxx                                                      #
#     21      10xxxxxx    10xxxxxx                                          #
#     26      10xxxxxx    10xxxxxx    10xxxxxx                              #
#     31      10xxxxxx    10xxxxxx    10xxxxxx    10xxxxxx                  #
#                                                                           #
# The original specification covered numbers up to 31 bits (the             #
# original limit of the Universal Character Set). In November 2003          #
# UTF-8 was restricted by RFC 3629 to end at U+10FFFF, in order to          #
# match the constraints of the UTF-16 character encoding. This removed      #
# all 5- and 6-byte sequences, and about half of the 4-byte sequences.      #
#                                                                           #
# The salient features of this scheme are as follows:                       #
#                                                                           #
#  One-byte codes are used only for the ASCII values 0 through 127.         #
#  In this case the UTF-8 code has the same value as the ASCII code.        #
#  The high-order bit of these codes is always 0.                           #
#                                                                           #
#  Code points larger than 127 are represented by multi-byte                #
#  sequences, composed of a leading byte and one or more continuation       #
#  bytes. The leading byte has two or more high-order 1s followed by a      #
#  0, while continuation bytes all have '10' in the high-order              #
#  position.                                                                #
#                                                                           #
#  The number of high-order 1s in the leading byte of a multi-byte          #
#  sequence indicates the number of bytes in the sequence, so that the      #
#  length of the sequence can be determined without examining the           #
#  continuation bytes.                                                      #
#                                                                           #
#  The remaining bits of the encoding are used for the bits of the          #
#  code point being encoded, padded with high-order 0s if necessary.        #
#  The high-order bits go in the lead byte, lower-order bits in             #
#  succeeding continuation bytes. The number of bytes in the encoding       #
#  is the minimum required to hold all the significant bits of the          #
#  code point.                                                              #
#                                                                           #
#  Single bytes, leading bytes, and continuation bytes do not share         #
#  values. This makes the scheme self-synchronising, allowing the           #
#  start of a character to be found by backing up at most five bytes        #
#  (three bytes in actual UTF‑8 as explained below).                        #
#                                                                           #
# The first 128 characters (US-ASCII) need one byte. The next 1,920         #
# characters need two bytes to encode. This covers the remainder of         #
# almost all Latin-derived alphabets, and also Greek, Cyrillic,             #
# Coptic, Armenian, Hebrew, Arabic, Syriac and Tāna alphabets, as well      #
# as Combining Diacritical Marks. Three bytes are needed for                #
# characters in the rest of the Basic Multilingual Plane (which             #
# contains virtually all characters in common use[12]). Four bytes are      #
# needed for characters in the other planes of Unicode, which include       #
# less common CJK characters and various historic scripts and               #
# mathematical symbols.                                                     #
#                                                                           #
# Examples:                                                                 #
#                                                                           #
# Let us consider how to encode the Euro sign, €.                           #
#                                                                           #
#  The Unicode code point for "€" is U+20AC.                                #
#  According to the scheme table above, this will take three bytes to       #
#  encode, since it is between U+0800 and U+FFFF.                           #
#                                                                           #
#  Hexadecimal 20AC is binary 0010000010101100. The two leading zeros       #
#  are added because, as the scheme table shows, a three-byte encoding      #
#  needs exactly sixteen bits from the code point.                          #
#                                                                           #
#  Because it is a three-byte encoding, the leading byte starts with        #
#  three 1s, then a 0 (1110...)                                             #
#                                                                           #
#  The remaining bits of this byte are taken from the code point            #
#  (11100010), leaving ...000010101100.                                     #
#                                                                           #
#  Each of the continuation bytes starts with 10 and takes six bits of      #
#  the code point (so 10000010, then 10101100).                             #
#                                                                           #
# The three bytes 11100010 10000010 10101100 can be more concisely          #
# written in hexadecimal, as E2 82 AC.                                      #
#                                                                           #
# The following table summarises this conversion, as well as others         #
# with different lengths in UTF-8. The colours indicate how bits from       #
# the code point are distributed among the UTF-8 bytes. Additional          #
# bits added by the UTF-8 encoding process are shown in black.              #
#                                                                           #
# Character   Binary code     Binary      Hexadecimal                       #
#             point           UTF-8       UTF-8                             #
#                                                                           #
# $   U+0024  0100100         00100100        24                            #
#                                                                           #
# ¢   U+00A2  000             11000010        C2 A2                         #
#             10100010        10100010                                      #
#                                                                           #
# €   U+20AC  00100000        11100010        E2 82 AC                      #
#             10101100        10000010                                      #
#                             10101100                                      #
#                                                                           #
# 𤭢 U+24B62 00010           11110000        F0 A4 AD A2                   #
#             01001011        10100100                                      #
#             01100010        10101101                                      #
#                             10100010                                      #
#                                                                           #
#############################################################################
#                                                                           #
# Conventions:                                                              #
#    Private fields start with __                                           #
#    Private methods start with _ or __                                     #
#                                                                           #
#    Private classes start with __                                          #
#                                                                           #
#    Private classes have an associated Public Class. When this occurs, the #
#    Public Class acts as the type and the Private Class acts as the values #
#    of that type. All such Public Classes have a common infrastructure and #
#    a common value generator method called New. The method creates a new   #
#    value by calling the initially method of the Private Class.            #
#                                                                           #
#    In addition, all such values have automatically associated a set of    #
#    methods that can be used for the Operator Overload facilities, if these#
#    have been compiled into the runtime system. These are made available   #
#    by defining a set of standard methods which can be used either directly#
#    as a method call or indirectly by using the specific Unicon operators  #
#    which calls the internally defined method which calls these publicly   #
#    available methods.                                                     #
#                                                                           #
#    This association is described in the file "README.txt" that is in this #
#    directory. The specific set of available operators is dependent on the #
#    associated public methods being defined for the relevant classes. If   #
#    no such method is defined, the relevant operator overload internally   #
#    defined method will either cause the operator to fail or produce a     #
#    runtime abort if these facilities are not defined in the runtime.      #
#                                                                           #
#                                                                           #
#############################################################################
#                                                                           #
# Public Classes:                                                           #
#                                                                           #
#    class UTF8 : ClassClass(                                               #
#                   __UTF8SizeTable,                                        #
#                   __UTF8LimitTable,                                       #
#                   __UTF8BOM,                                              #
#                   __UTF8CountTable,                                       #
#                   __ByteTranslate,                                        #
#                   __UTF8InvalidChar,                                      #
#                   __UTF8Max,                                              #
#                   __SpecialUTF8Strings                                    #
#           )                                                               #
#                                                                           #
#    class UTF8Set : ClassClass(                                            #
#                   __ascii_blank,                                          #
#                   __uft8_blanks,                                          #
#                   __all_codepoints,                                       #
#                   __ascii_lparens,                                        #
#                   __ascii_rparens,                                        #
#                   __ascii_128,                                            #
#                   __defined_sets,                                         #
#                   __max_size_of_set                                       #
#           )                                                               #
#                                                                           #
#############################################################################
#                                                                           #
# Private Classes:                                                          #
#                                                                           #
#    class __UTF8Object : ClassObject(                                      #
#                   __data,                                                 #
#                   __asciionly,                                            #
#                   __size,                                                 #
#                   __source_data                                           #
#           )                                                               #
#                                                                           #
#    class __UTF8SetObject : ClassObject(                                   #
#                   __allcp,                                                #
#                   __cset,                                                 #
#                   __2bytedata,                                            #
#                   __3bytedata,                                            #
#                   __4bytedata,                                            #
#                   __negated,                                              #
#                   __string_representation                                 #
#           )                                                               #
#                                                                           #
#############################################################################
#                                                                           #
# Debugging the Classes and Methods:                                        #
#                                                                           #
#   The superclass ErrorSystem provides a set of debugging facilities that  #
#   can be dynamically turned at runtime. The source code has many of the   #
#   debugging calls placed throughout.                                      #
#                                                                           #
#   When the debugging facilities are turned off, these calls are null      #
#   actions, even though the parameters are still evaluated. Nothing is     #
#   printed during the run.                                                 #
#                                                                           #
#############################################################################
#                                                                           #
#                                                                           #
#                                                                           #
#                                                                           #
#############################################################################
#                                                                           #
# Private Methods:                                                          #
#                                                                           #
#  class UTF8:                                                              #
#                                                                           #
#    method __Initialise()                                                  #
#    method __ActualPos(ustr, i)                                            #
#    method __Coerce(val)                                                   #
#    method __GetAt(ustr, i, j)                                             #
#    method __AuxMessage(errorno, messageno)                                #
#                                                                           #
#                                                                           #
#  class __UTF8Object:                                                      #
#                                                                           #
#                                                                           #
#  class UTF8Set:                                                           #
#                                                                           #
#    method __Coerce(val)                                                   #
#    method __AuxMessage(errorno, messageno)                                #
#                                                                           #
#                                                                           #
#  class __UTF8SetObject:                                                   #
#                                                                           #
#    method __DebugPrint(modifier, val, uset)                               #
#                                                                           #
#############################################################################
#                                                                           #
# Public Methods:                                                           #
#                                                                           #
#  class UTF8:                                                              #
#                                                                           #
#    ----- Testing method specific to UTF8 sequences                        #
#                                                                           #
#    method BOMFound(str)                                                   #
#    method SkipBOM(str)                                                    #
#    method Valid(str)                                                      #
#    method ByteTranslate(byte)                                             #
#    method Multibyte(codepoint)                                            #
#    method SpecialString(str)                                              #
#                                                                           #
#    ----- Translation methods                                              #
#                                                                           #
#    method ToUnicode32(ustr)                                               #
#    method Unicode32To(unicode)                                            #
#    method HexTo(hexcode)                                                  #
#    method DecimalTo(num)                                                  #
#                                                                           #
#    ----- Auxilary methods                                                 #
#                                                                           #
#    method AmpPos()                                                        #
#                                                                           #
#    ----- class methods equivalent to Operator Overload methods available  #
#               to object values                                            #
#                                                                           #
#    method Equiv(val1, val2)                                               #
#    method Nequiv(val1, val2)                                              #
#    method LexEquals(str1, str2)                                           #
#    method LexGTorEq(str1, str2)                                           #
#    method LexGT(str1, str2)                                               #
#    method LexLTorEq(str1, str2)                                           #
#    method LexLT(str1, str2)                                               #
#    method LexNE(str1, str2)                                               #
#    method Concatenate(ustr1, ustr2)                                       #
#    method Complement(val)                                                 #
#    method Difference(val1, val2)                                          #
#    method Intersect(val1, val2)                                           #
#    method Union(val1, val2)                                               #
#    method ForEach(str)                                                    #
#    method Size(str)                                                       #
#    method Random(str)                                                     #
#    method Subsection(ustr, i, j)                                          #
#    method Subscript(str, i)                                               #
#    method TabMatch(str)                                                   #
#                                                                           #
#    ----- UTF8 analogues to standard Unicon string functions               #
#                                                                           #
#    method Any(uset, ustr, i, j)                                           #
#    method Bal(uset, usetopen, usetclose, ustr, i, j)                      #
#    method Center(ustr, i, pstr)                                           #
#    method Char(i)                                                         #
#    method Detab(ustr, i[])                                                #
#    method Entab(ustr, i[])                                                #
#    method Find(ustr1, ustr, i, j)                                         #
#    method Left(ustr, i, pstr)                                             #
#    method Many(uset, ustr, i, j)                                          #
#    method Map(ustr, ustr1, ustr2)                                         #
#    method Match(ustr1, ustr, i, j)                                        #
#    method Move(i)                                                         #
#    method Ord(ustr)                                                       #
#    method Pos(i)                                                          #
#    method Repl(ustr, i)                                                   #
#    method Reverse(ustr)                                                   #
#    method Right(ustr, i, pstr)                                            #
#    method String(val)                                                     #
#    method Tab(i)                                                          #
#    method Trim(ustr, uset, i, allutf8blanks)                              #
#    method Upto(uset, ustr, i, j)                                          #
#                                                                           #
#                                                                           #
#  class __UTF8Object:                                                      #
#                                                                           #
#    ----- Auxiliary Data methods                                           #
#                                                                           #
#    method Data()                                                          #
#    method Invalid()                                                       #
#                                                                           #
#    ----- Operator Overload methods                                        #
#                                                                           #
#    method Complement()                                                    #
#    method Difference(val)                                                 #
#    method Intersect(val)                                                  #
#    method Union(val)                                                      #
#    method Equiv(val)                                                      #
#    method Nequiv(val)                                                     #
#    method LexEquals(str)                                                  #
#    method LexGTorEq(str)                                                  #
#    method LexGT(str)                                                      #
#    method LexLTorEq(str)                                                  #
#    method LexLT(str)                                                      #
#    method LexNE(str)                                                      #
#    method ForEach()                                                       #
#    method Random()                                                        #
#    method TabMatch()                                                      #
#    method Size()                                                          #
#    method Concatenate(ustr)                                               #
#    method Subsection(i, j)                                                #
#    method Subscript(i)                                                    #
#                                                                           #
#    ----- UTF8 analogues to standard Unicon string functions               #
#                                                                           #
#    method String()                                                        #
#                                                                           #
#  class UTF8Set:                                                           #
#                                                                           #
#    ----- class methods equivalent to Operator Overload methods available  #
#               to object values                                            #
#                                                                           #
#    method Complement(uset)                                                #
#    method Difference(uset1, uset2)                                        #
#    method Intersect(uset1, uset2)                                         #
#    method Union(uset1, uset2)                                             #
#    method ForEach(val)                                                    #
#    method Size(val)                                                       #
#    method Subscript(val, i)                                               #
#                                                                           #
#    ----- UTF8Set analogues to standard Unicon string functions            #
#                                                                           #
#    method String(val)                                                     #
#    method Member(uset, val[])                                             #
#                                                                           #
#    ----- Auxilary methods                                                 #
#                                                                           #
#    method Space()                                                         #
#    method UTF8Spaces()                                                    #
#    method Ascii()                                                         #
#    method Rparens()                                                       #
#    method Lparens()                                                       #
#    method All()                                                           #
#    method MaxSize()                                                       #
#    method Defined(str)                                                    #
#                                                                           #
#  class __UTF8SetObject:                                                   #
#                                                                           #
#    ----- Operator Overload methods                                        #
#                                                                           #
#    method Equiv(uset)                                                     #
#    method Nequiv(uset)                                                    #
#    method LexEquals(str)                                                  #
#    method LexGTorEq(str)                                                  #
#    method LexGT(str)                                                      #
#    method LexLTorEq(str)                                                  #
#    method LexLT(str)                                                      #
#    method LexNE(str)                                                      #
#    method Complement()                                                    #
#    method Difference(val)                                                 #
#    method Intersect(val)                                                  #
#    method Union(val)                                                      #
#    method ForEach()                                                       #
#    method Size()                                                          #
#    method Subscript(i)                                                    #
#    method Random()                                                        #
#    method TabMatch()                                                      #
#    method Concatenate(ustr)                                               #
#    method Subsection(i, j)                                                #
#                                                                           #
#    ----- UTF8 analogues to standard Unicon string functions               #
#                                                                           #
#    method Member(c[])                                                     #
#    method String()                                                        #
#                                                                           #
#    ----- Auxiliary Data methods                                           #
#                                                                           #
#    method UTF8Value()                                                     #
#                                                                           #
#                                                                           #
#                                                                           #
#############################################################################
#                                                                           #
# Public Procedures:                                                        #
#                                                                           #
#############################################################################

#### Completed ####
#IND:
#: required include files

#### Completed ####
#LKD:
#: required link statements
link ximage                             #LK: IPL file - functionality to produce
                                        #:   string images of all Unicon values

link hexcvt                             #LK: IPL file - functionality to convert
                                        #:   strings of hex digits to and from
                                        #:   integer values

link equiv                              #LK: IPL file - functionality to test if
                                        #:   two objects or structures are equivalent
                                        #:   in value

#### Completed ####
#IMD:
#: required import statements

import ClassObject                      #IM: Locally defined Unicon class file
                                        #:   which contains the basic facilities
                                        #:   used by all classes in my development
                                        #:   hierarchy

#### Completed ####
#PKD:
#: Package UTF8 - This is the package that implements the basic processing for UTF-8
#: strings. The classes defined in this package and the associated objects/values
#: generated by these classes will have corresponding methods that match the
#: standard strings functions provided by Unicon/Icon.
#:
#: Each class defined in this package inherits a standard set of fields and methods
#: from the ClassObject and ClassClass classes defined in the package ClassObject.
#: The inherited class hierarchy and method functionality provide the basic
#: requirements in accordance with the new class/object model that I am using.
#:
#: In essence, the class UTF8 acts as the type and the associated class __UTF8Object
#: acts as the value generator for this type. The same is applicable for class
#: UTF8Set which acts as the type for the associated objects created with class
#: __UTF8SetObject.
#:
#: Each kind of class created under this development model will be in its own
#: package to ensure that there is no crossover or interference with other
#: packages or classes to be found in the standard distributed Unicon Class Libraries.
#:
#: Therefore this package will be called UTF8 to distinguish it from all other
#: facilitities.
#:
package UTF8

#### Completed ####
#GBL:
#: Define the required global statements for use within this package.
#:
#: There are no global variables required or created in this package.


#### Completed ####
#CD: class UTF8 - This class is a singleton object and acts as the type for all
#: UTF-8 objects that can be generated from this packages functionality.
#:
#: It is used to create each UTF-8 value via the method New(), which will use
#: the class __UTF8Object to create the actual value required.
#:
class UTF8 : ClassClass(
    __UTF8SizeTable,                    #CV: This is an Unicon table whose keys
                                        #:   are each of the 256 8 bit bytes
                                        #:   available within the Unicon system.
                                        #:   The value for each key will either
                                        #:   be the number of bytes that will
                                        #:   be found in a valid UTF-8 codepoint,
                                        #:   or the value 0 which indicates the
                                        #:   byte is an invalid starting byte,
                                        #:   or it will be -1 which designates
                                        #:   that the byte is an internal extension
                                        #:   byte for any UTF-8 codepoint.

    __UTF8LimitTable,                   #CV: This is an Unicon table whose keys
                                        #:   are the number of possible bytes in
                                        #:   a UTF-8 codepoint, which at time is
                                        #:   limited to the keys of 1, 2, 3 and 4.
                                        #:   The values for each key is a list of
                                        #:   lists. Each list will contain the
                                        #:   number of elements as per the key
                                        #:   value and each element is a list of
                                        #:   2 elements, the first is the starting
                                        #:   byte value and second element is
                                        #:   the ending byte value applicable
                                        #:   for each byte position in the UTF-8
                                        #:   codepoint.

    __UTF8BOM,                          #CV: This is an UTF-8 value object and
                                        #:   not a Unicon string. It represents
                                        #:   the optional byte string that can
                                        #:   be at the start of a UTF-8 file.

    __UTF8CountTable,                   #CV: This is an Unicon table whose keys
                                        #:   are each of the 256 8 bit bytes
                                        #:   available within the Unicon system.
                                        #:   The value for each key will either
                                        #:   be a 1 if the byte is to be
                                        #:   counted as the start of a valid UTF-8
                                        #:   codepoint or a 0 if it is not. This
                                        #:   allows a simple way to count the
                                        #:   actual number of UTF-8 codepoints
                                        #:   within a UTF-8 stream.

    __ByteTranslate,                    #CV: This is an Unicon table and contains
                                        #:   a translation of a binary byte value
                                        #:   to the corresponding UTF-8 byte
                                        #:   sequence.

    __UTF8InvalidChar,                  #CV: This is an UTF-8 value object and
                                        #:   not a Unicon string. It represents
                                        #:   the invalid byte string to be used
                                        #:   when invalid bytes are found in the
                                        #:   UTF-8 string when being processed.

    __UTF8Max,                          #CV: This holds the maximum integer value that
                                        #:   represents a valid UTF-8 codepoint.
                                        #:   At this point, this is integer
                                        #:   representation of the hex value
                                        #:   U+10FFFF.

    __SpecialUTF8Strings                #CV: This holds a table of special UTF-8 codepoint strings
                                        #:   that really only need to be created
                                        #:   once. This includes each of the
                                        #:   printable ASCII characters, the entire
                                        #:   128 character ASCII string. It can
                                        #:   be added to by a special method. This
                                        #:   will allow the same object to be
                                        #:   returned if it occurs many times,
                                        #:   instead of a new UTF-8 object being
                                        #:   each time for it. It is keyed by the
                                        #:   raw binary data of the UTF-8 codepoint strings.
)

    #MD:
    #: __Initialise() - this is a private method and should only ever be run once.
    #: To ensure this can only be run once, the entire code in the procedure is
    #: placed in the initial {} expression, so that any inadvertent or even
    #: deliberate attempts to run this method will be essentially null operations
    #: after the class is initially called.
    #:
    #:@params           none
    #:@returns          always fails
    #:
    method __Initialise()
        local   charlist,               #LV: a string containing all available bytes
                i                       #LV: local counting variable

        initial {
            #
            # we want a string that contains each ASCII character in the underlying
            # unicon/icon virtual machine. This string will have the full 256
            # characters available in unicon/icon.
            #
            charlist := ::string(&cset)

            #
            # table holding code point sizes, continuation bytes marked as -1,
            # invalid bytes as 0
            #
            __UTF8SizeTable := ::table()

            #
            # initialise this table to a consistent state for all ASCII byte values.
            # the subsequent processing will set each range of characters to the
            # relevant code point size. At this point, we only allow up to 4
            # bytes to be available for any UTF-8 codepoint.
            #
            # Note: not all 4 byte sequences are valid - see the details that are
            # described in the main description given above for the additional
            # information. To simplify the initialisation, we set all bytes to
            # a value of 0 and then set the specific ranges to the correct values.
            # This will ensure that all invalid bytes will be set properly.
            #
            every __UTF8SizeTable[charlist[1 to 256]] := 0       # \x00 -> \xFF

            #
            # These bytes are for the ASCII values (1 byte encodings)
            #
            every __UTF8SizeTable[charlist[1 to 128]] := 1       # \x00 -> \x7F

            #
            # all continuation bytes will have values in this range only
            #
            every __UTF8SizeTable[charlist[129 to 192]] := -1    # \x80 -> \xBF

            #
            # These are the start bytes for 2 byte encodings
            #
            every __UTF8SizeTable[charlist[195 to 224]] := 2     # \xC2 -> \xDF

            #
            # These are the start bytes for 3 byte encodings
            #
            every __UTF8SizeTable[charlist[225 to 240]] := 3     # \xE0 -> \xEF

            #
            # These are the start bytes for 4 byte encodings
            #
            every __UTF8SizeTable[charlist[241 to 245]] := 4     # \xF0 -> \xF5

            #
            # used for counting UTF-8 characters in string, those ASCII bytes that
            # do not contribute to a valid code point will have a count set to 0,
            # this covers all bytes in the continuation bytes range and invalid
            # bytes ranges. So that only the start byte is significant in counting
            # UTF-8 codepoints in a UTF-8 string
            #
            __UTF8CountTable := ::table()
            every __UTF8CountTable[charlist[1 to 256]] := 0      # \x00 -> \xFF
            every __UTF8CountTable[charlist[1 to 128]] := 1      # \x00 -> \x7F
            every __UTF8CountTable[charlist[195 to 224]] := 1    # \xC2 -> \xDF
            every __UTF8CountTable[charlist[225 to 240]] := 1    # \xE0 -> \xEF
            every __UTF8CountTable[charlist[241 to 245]] := 1    # \xF0 -> \xF5

            #
            # used for validation of bytes found for UTF-8 code points, each list
            # value is a list of lists, where the inner lists specify the start
            # and end valid values for that byte position in the code point
            #
            # PLEASE NOTE: That this is an approximation, as there are a number
            # of specific code points that are actually invalid, even though these
            # code points will fit into the limits given below. These are dealt
            # with as special cases in the appropriate procedures below.
            #
            # An example of one of these code points is: ################ Fix this detail
            #
            __UTF8LimitTable := ::table()
            __UTF8LimitTable[1] := [["\x00", "\x7F"]]
            __UTF8LimitTable[2] := [["\xC2", "\xDF"], ["\x80", "\xBF"]]
            __UTF8LimitTable[3] := [["\xE0", "\xEF"], ["\x80", "\xBF"], ["\x80", "\xBF"]]
            __UTF8LimitTable[4] := [["\xF0", "\xF5"], ["\x80", "\xBF"], ["\x80", "\xBF"], ["\x80", "\xBF"]]

            #
            # Initialise the __SpecialUTF8Strings to hold a table
            #
            __SpecialUTF8Strings := ::table()
            charlist := ::string(&ascii)
            every i := !charlist do {
                __SpecialUTF8Strings[i] := UTF8::UTF8.New(i)
            }
            __SpecialUTF8Strings[charlist] := UTF8::UTF8.New(charlist)

            #
            # UTF-8 Byte Order Mark, optional value that can appear at start of
            # UTF-8 file. This is an otherwise invalid combination for a UTF-8
            # codepoint and should never be found elsewhere in a UTF-8 byte stream.
            #
            __UTF8BOM := SpecialString("\xEF\xBB\xBF")

            DebugPrint("__UTF8BOM:", ::image(__UTF8BOM))

            #
            # This encoding will be used to replace any invalid byte sequences
            #
            __UTF8InvalidChar := SpecialString("\xEF\xBF\xBD")

            DebugPrint("__UTF8InvalidChar:", ::image(__UTF8InvalidChar))

            #
            # we want a string that contains each ASCII character in the underlying
            # unicon/icon virtual machine. This string will have the full 256
            # characters available in unicon/icon.
            #
            charlist := ::string(&cset)

            #
            # Since only half of the full byte values are actual ASCII, the other
            # half has the highest order bit is set and this will be represented
            # by a two byte sequence as a UTF-8 codepoint.
            #
            DebugPrint("Creating the Byte Translate table")

            __ByteTranslate := ::table()

            #
            # translate the first 128 7 bit ASCII (\x00 to \x7F) to corresponding
            # 7 bit ASCII
            #
            every __ByteTranslate[charlist[i := 1 to 128]] := charlist[i]

            #
            # for the next 64 byte values (\x80 to \xBF) need the leading byte to
            # be \xC2 and the following byte to be from \x80 to \xBF
            #
            every __ByteTranslate[charlist[i := 129 to 192]] := "\xC2" || char(i - 1)

            #
            # for the final 64 byte values (\xC0 to \xFF) need the leading byte to
            # be \xC3 and the following byte to be from \x80 to \xBF
            #
            every __ByteTranslate[charlist[i := 193 to 256]] := "\xC3" || char(i - 65)

            DebugPrint("Creating the Byte Translate table has finished")

            #
            # Initialise the maximum value for a UTF-8 codepoint. If the standard
            # changes and allows more encoding sequences, this is the value that
            # will need to be changed accordingly.
            #
            __UTF8Max := ::hex("10FFFF")
        }
    end

    ###########################################################################
    #
    # The following methods are internal helper functions for this class. They
    # are only to used by the public methods.
    #
    #   Method Name                             Returns/Suspends
    #
    #   __ActualPos                             integer
    #   __Coerce                                UTF-8 string object
    #   __GetAt                                 UTF-8 string object
    #   __AuxMessage                            error message text based on error
    #                                           number and message number
    #
    ###########################################################################

    #### Completed ####
    #MD:
    #: __ActualPos(ustr, i) - this is a class private method and is used to
    #: translate the pos specified by i within the specified UTF-8 string ustr
    #: to the actual pos as expected if we were treating the UTF-8 string data
    #: as a normal unicon/icon string. This method does not do any coercion to
    #: to a UTF-8 string as it expects this to have already been done earlier
    #: in any procedure or method that calls this method. As it is private, this
    #: is a reasonable assumption.
    #:
    #:@param ustr       the UTF-8 string we are looking at
    #:@param i          the position in the UTF-8 string that we are interested
    #:@                 in. This position treats multi-byte codepoint as if they
    #:@                 are an indivisible unit. If it is negative or 0, it is
    #:@                 referring to the RHS of the supplied string.
    #:@returns          the actual position of the first byte of the codepoint
    #:@                 as it would be if the UTF-8 string was treated as a normal
    #:@                 ASCII byte string
    #:
    method __ActualPos(ustr, i)
        local   j := 1,                 #LV: we initialise this to the start
                                        #:   position in the string relative
                                        #:   to the LHS

                k := 0,                 #LV:

                data,                   #LV: temporary to hold the actual string
                                        #:   data of the supplied UTF-8 object

                s                       #LV: temporary to hold the size of the
                                        #:   supplied UTF-8 object in codepoints

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          __AuxMessage(101))

        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        #
        # we assume that the supplied string is always going to be an actual
        # UTF-8 object. So we just select out the actual data associated with
        # the UTF-8 object.
        #
        data := ustr.String()

        DebugPrint("data:", ::ximage(data))

        #
        # we need to have the size of the UTF-8 string in codepoints for the
        # translation of codepoint position to byte position in the string
        #
        s := ustr.Size()

        DebugPrint("s:", s, "\n", "i:", i)

        #
        # if the position has been specified as less than or equal to 0, we need to
        # convert this value to the position as referenced from the start (LHS)
        # of the string. This is where we need to know the size of the UTF-8
        # string
        #
        (i < 1) & i := s + 1 + i

        DebugPrint("i:", i)

        #
        #
        # To determine the real byte position, we need to step through each
        # codepoint. We have available to us a table which has each valid byte
        # that can be found in any UTF-8 codepoint and which has a count of 1
        # for each starting byte of a codepoint and has a count of 0 for each
        # subsequent byte of a multi-byte codepoint.
        #
        # In effect, we use the table to determine how many codepoints we have
        # seen and we count the actual number of bytes we have passed.
        # so k count of bytes, j count of codepoints
        #
        while j <= i & k <= *data do {
            k +:= 1
            j +:= __UTF8CountTable[data[k]]
        }

        DebugPrint("i:", i, "\n", "j:", j, "\n", "k:", k)

        return k
    end

    #MD:
    #: __Coerce(val) - internal method to transform a supplied value into a
    #: UTF-8 string object if the transformation is required, otherwise if it is
    #: already a UTF-8 string object, just return that supplied value
    #:
    #:@param            any unicon value of object
    #:@returns          if it is already an UTF-8 string object, just return
    #:@                 the supplied value, otherwise try and create a new
    #:@                 UTF-8 string object from the supplied value
    #:@aborts           will occur if the supplied value cannot be converted
    #:@                 to a UTF-8 string object
    #:
    method __Coerce(val)
        DebugPrint("level of __Coerce(", &level, ") type of val: ", ::type(val),
                   " value of val: ", ::image(val))
        DebugPrint("level of __Coerce(", &level, ") class of val: ", ::classname(val) | "none")

        /val & AbortMessage(2000, keyword("&file", , 1), keyword("&line", , 1),
                                        __AuxMessage(2000))

        if IsObjectOf(val, "UTF8") then {
            DebugPrint("return val:", ::image(val))

            return val
        } else {
            DebugPrint("creating a New UTF-8 value")

            return UTF8::UTF8.New(val)
        }
    end

    #### Completed ####
    #MD:
    #: __GetAt(ustr, i, j) - internal method to get the UTF-8 codepoints between
    #: the positions i and j. These two integer values are the positions based on
    #: codepoints. They will get converted to the actual byte positions.
    #:
    #: the value of ustr is assumed to be a UTF-8 string object. This is a valid
    #: assumption as in all places that it is used internally to the class, the
    #: supplied value is already a coerced into a UTF-8 string object.
    #:
    #:@param ustr       this is a UTF-8 string object
    #:@param i          the position in the UTF-8 string that we are interested
    #:@                 in starting at. This position treats multi-byte codepoint
    #:@                 as if they are an indivisible unit. If it is negative or 0,
    #:@                 it is referring to the RHS of the supplied string.
    #:@param j          the position in the UTF-8 string that we are interested
    #:@                 in finishing at. This position treats multi-byte codepoint
    #:@                 as if they are an indivisible unit. If it is negative or 0,
    #:@                 it is referring to the RHS of the supplied string.
    #:@returns          a new UTF-8 string object containing the codepoints
    #:@                 between the specified endpoints
    #:
    method __GetAt(ustr, i, j)
        local   i1,                     #LV: temporary to hold the actual byte
                                        #:   position corresponding to i

                j1,                     #LV: temporary to hold the actual byte
                                        #:   position corresponding to j

                k,                      #LV: temporary to hold the size of the
                                        #:   supplied UTF-8 string object

                data                    #LV: temporary to hold the actual byte
                                        #:   data of the supplied UTF-8 string

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        DebugPrint("ustr:", ::image(ustr), " String():", ::image(ustr.String()),
                   " Size():", ustr.Size())

        #
        # we need the size of the supplied UTF-8 string object as we may need to
        # calculate the correct codepoint positions, if they were supplied starting
        # from the RHS, i.e., the positions are less than 0
        #
        k := ustr.Size()

        #
        # the default if no parameter is supplied will be 1 (as per all of the
        # normal unicon/icon string functions) for position i
        #
        /i := 1

        #
        # we convert a negative i to a position from the LHS of the string. This
        # needs to be done before doing any calculation on j when j is &null. If
        # the calculation done after setting j becasue it is &null, we can get
        # the the entire string being returned instead of a single codepoint.
        # This situation will occur if the value of i = 0
        #
        (i <= 0) & i := k + i + 1

        #
        # the default if no parameter is supplied will be the position following
        # whatever value is now in i. This means that only one codepoint is being
        # returned
        #
        /j := i + 1

        #
        # we convert any negative j to a position from the LHS of the string
        #
        (j <= 0) & j := k + j + 1

        #
        # if i is greater than j we need to swap the values
        #
        (i > j) & (i :=:j)

        #
        # if i is equal to j, there is nothing to do and we should fail
        #
        (i = j) & fail


        DebugPrint("k:", k, "\n", "j:", j, "\n", "i:", i)

        #
        # At this point both i and j must be between the starting and ending
        # positions of the UTF-8 string, otherwise we want to fail
        #
        ((i | j) <= (k + 1)) | fail

        #
        # we now need to find the actual position in terms of bytes (unicon
        # characters) for both i and j, we will store each of these positions
        # in temporary variables for later extraction of the required bytes
        #
        i1 := __ActualPos(ustr, i)
        j1 := __ActualPos(ustr, j)

        DebugPrint("i1:", i1, "\n", "j1:", j1)

        #
        # we now need the raw data from the UTF-8 string object, from which we
        # will extract the required bytes
        #
        data := ustr.String()

        DebugPrint("data:", ::ximage(data), "\n", "*data:", *data, "\n",
                   "i1:", i1, "\n", "j1:", j1)

        #
        # grab the sub-string required, we use this to create a new UTF-8 string
        #
        data := data[i1:j1]

        DebugPrint("data:", ::ximage(data))

        #
        # using the selected bytes, we will return a new UTF-8 string object
        # to th calling expression
        #
        return UTF8::UTF8.New(data)
    end

    #MD:
    #: __AuxMessage(errorno, messageno)
    #:
    #:
    method __AuxMessage(errorno, messageno)
        static  ErrorMessages           #SV: table holding error messages for error
                                        #:   numbers used in UFT8/UTF8Set aborts

        initial {
            ErrorMessages := ::table()

            ErrorMessages[103 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8 object/value or object/value that can be converted to an UTF8 object/value."
            ErrorMessages[103 || "-" || 2] := "\noffending value: "
            ErrorMessages[103 || "-" || 3] := "\noffending value: &null_
                                               \nMissing Unicon string value"
            ErrorMessages[103 || "-" || 4] := "\noffending value: &null_
                                               \nMissing Unicon integer value or Unicon string value"
            ErrorMessages[104 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[104 || "-" || 2] := "\noffending value: &null"
            ErrorMessages[110 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8 object/value or object/value that can be converted to an UTF8 object/value."
            ErrorMessages[113 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8 object/value or object/value that can be converted to an UTF8 object/value."
            ErrorMessages[120 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[2000 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8 object/value or object/value that can be converted to an UTF8 object/value."
        }
        /messageno := 1
        return \ErrorMessages[errorno || "-" || messageno] | \(self$ClassClass.__AuxMessage(errorno, messageno))
    end

    #### Completed ####
    #MD:
    #: BOMFound(str) - given any value that can be coerced to a UTF-8 sequence,
    #: this will test that sequence for an initial Byte order mark value and
    #: return the UTF-8 Byte Order Mark as a UTF-8 value.
    #:
    #:@param str        str - this can be any value that can be coerced into a
    #:@                 UTF-8 sequence (string). Typical use will be a UTF-8
    #:@                 sequence
    #:@returns          if the value of __UTF8BOM is found at the start then we
    #:@                 will return this UTF-8 object
    #:@fails            fails if no byte order mark is found
    #:@aborts           if a value cannot be coerced to a UTF-8 object (103 - not a string)
    #:
    method BOMFound(str)
        local   str1                    #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(103))

        #
        # coerce the value supplied into a UTF-8 string object. The coerced value
        # is placed in a new variable to ensure that the original value/variable
        # supplied is not altered. If cannot be coerced then a runtime abort
        # is performed with a value of 103 - "not a string"
        #
        str1 := __Coerce(str)

        #
        # return the byte order mark as a UTF-8 object value by checking the str byte
        # data against the BOM byte data, otherwise fail
        #
        return str1.String() ? (=__UTF8BOM.String() & __UTF8BOM)
    end

    #### Completed ####
    #MD:
    #: SkipBOM(str) - this will return a new UTF-8 object. If the original value
    #: supplied has a byte order mark, this BOM will be removed first and the
    #: resultant string will be used as the data for the new UTF-8 object. If
    #: there is no byte order mark, then the original data is used as the value
    #: for the new UTF-8 object. Again, if a coercion is needed this is will be
    #: the first action done.
    #:
    #:@param str        any value, though usually a unicon string or a UTF-8
    #:@                 object
    #:@returns          a UTF-8 object that has had any byte order mark removed. If
    #:@                 the original supplied UTF-8 string has no BOm, then it is
    #:@                 returned
    #:@aborts           if a value cannot be coerced to a UTF-8 object (103 - not a string)
    #:
    method SkipBOM(str)
        local   str1,                   #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                str2                    #LV: temporary to hold the result data
                                        #:   sans BOM. This is an Unicon string value

        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(103))

        #
        # coerce the value supplied into a UTF-8 object. The coerced value
        # is placed in a new variable to ensure that the original value/variable
        # supplied is not altered. If cannot be coerced then a runtime abort
        # is performed with a value of 103 - "not a string"
        #
        str1 := __Coerce(str)

        DebugPrint("str1.String():", ::ximage(str1.String()))

        #
        # search for a byte order mark and skip if found, return the resultant data
        #
        str1.String() ? {
            #
            # This will match a BOM if it exists and updates the &pos in the
            # scanning environment. If it does not match, no update to &pos will
            # occur.
            #
            =(__UTF8BOM.String())

            #
            # This assignment will depend on the starting value which will be
            # found in &pos and the end of the string
            #
            str2 := ::tab(0)
        }

        DebugPrint("str2:", ::ximage(str2))

        #
        # return this data as a new UTF-8 string object to the calling expression.
        # This ensures that the immutable semantics for normal strings is the
        # same for UTF-8 string objects.
        #
        if str1.String() == str2 then {
            return str
        } else {
            return UTF8::UTF8.New(str2)
        }
    end

    ##########################
    ##########################
    ###
    ### Needs a fix for the specific invalid UTF-8 code points known in the upper
    ### reaches of the 3 and 4 byte code points
    ###
    ##########################
    ##########################
    #MD:
    #: Valid(str) - a procedure that tests a UTF-8 string for any invalid UTF-8
    #: characters. If none are found then the raw data is returned otherwise
    #: failure is signalled.
    #:
    #: If the value being tested cannot be coerced into a UTF-8 string, a runtime
    #: abort is performed with a value of 103 - "not a string"
    #:
    #:@param str        the value being tested for UTF-8 codepoints. If it is not
    #:@                 a UTF-8 object, an attempt to coerce it into one will be done
    #:@returns          if it contains only valid UTF-8 codepoints, the raw data
    #:@                 will returned
    #:@fails            if any sequence of bytes is not a valid codepoint
    #:
    method Valid(str)
        local   i,                      #LV: temporary to hold number of bytes
                                        #:   in the current codepoint being tested

                local_str,              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                byte,                   #LV: temporary to hold the current byte
                                        #:   being examined from the current
                                        #:   codepoint

                lowerbyte,              #LV: temporary to hold the current lowest
                                        #:   byte value for this position in the
                                        #:   current codepoint. This is compared
                                        #:   with the current byte being tested.

                upperbyte,              #LV: temporary to hold the current highest
                                        #:   byte value for this position in the
                                        #:   current codepoint. This is compared
                                        #:   with the current byte being tested.

                limits,                 #LV: temporary to hold the current limits
                                        #:   that are applicable for the currently
                                        #:   tested byte value in the currently
                                        #:   tested codepoint.

                j,                      #LV: temporary count variable

                data                    #LV: temporary to hold string ot UTF8 String()
                                        #:   data value

        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(103))


        DebugPrint("type of str: ", ::type(str), " value of str: ", ::image(str))

        data := if ::type(str) == "string" then {
            str
        } else if IsObjectOf(str, "UTF8") then {
            str.String()
        } else if IsObject(str, "UTF8Set") then {
            str.UTF8Value().String()
        } else if ::string(str) then {
            ::string(str)
        } else {
            AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                         __AuxMessage(103, 2), ::image(str))
        }

        DebugPrint("type of data: ", ::type(data), " value of data:", ::image(data))

        #
        # we will now scan and test each byte to ensure that it belongs to a
        # valid UTF-8 codepoint
        #
        data ? {

            DebugPrint("data:", ::ximage(data))

            #
            # while we have not reached the end of the byte string, we need to
            # check each codepoint and each byte within the codepoint for validity
            #
            while not ::pos(0) do {
                #
                # collect the next unexamined byte, this should be the first byte
                # of a valid UTF-8 codepoint. It will be the start of a 1 to 4
                # byte sequence.
                #
                byte := ::move(1)

                DebugPrint("byte:", ::ximage(byte))

                #
                # using the byte found, we obtain how big the codepoint in bytes
                # will be. We recognise that the byte we are currently examining
                # will be a start byte of some UTF-8 codepoint.
                #
                # However, it is possible that the byte may be an invalid codepoint
                # starting byte. We determine the validity of this by the value
                # found in the internal table __UTF8SizeTable. Any value returned
                # that is less than one, means that the byte being examined is
                # not a valid starting byte and we will cause the procedure to fail.
                # We don't need to specify the upper limit on the number of bytes
                # which is currently stands at four, as the invalid sizes are
                # either 0 or -1.
                #
                if (i := __UTF8SizeTable[byte]) >= 1 then {

                    DebugPrint("*:", i, ":*")

                    #
                    # each size that a UTF-8 codepoint can have has a list of
                    # byte value limits for each byte within the codepoint. We
                    # need to select these limits based on the expected UTF-8
                    # codepoint size
                    #
                    limits := __UTF8LimitTable[i]


                    DebugPrint("l:", ::ximage(limits), ":l")

                    #
                    # We select the upper and lower byte limits for the first
                    # byte to be examined.
                    #
                    lowerbyte := limits[1, 1]
                    upperbyte := limits[1, 2]

                    DebugPrint("lowerbyte:", ::image(lowerbyte), " upperbyte:", ::image(upperbyte))

                    #
                    # as long as the byte is between the specified range limits
                    # we can then continue onto the following bytes, testing each
                    # in turn. If an invalid byte is found, we will cause the
                    # method to fail
                    #
                    if lowerbyte <<= byte <<= upperbyte then {
                        every j := 2 to i do {
                            byte := ::move(1)
                            lowerbyte := limits[j, 1]
                            upperbyte := limits[j, 2]
                            if not (lowerbyte <<= byte <<= upperbyte) then {
                                fail
                            }
                        }

                    #
                    # the byte examined is outside the valid range for this length
                    # of codepoint bytes (that is 0 or -1), hence we must fail
                    # under this condition
                    #
                    } else {
                        fail
                    }

                #
                # An invalid starting byte has been found, this procedure must fail
                # under this condition
                #
                } else {
                    fail
                }
            }

            #
            # we only reach this point, if all codepoints found in the source are
            # valid UTF-8 codepoints. we return the UTF-8 string object we created
            # at the start with the coercion.
            #
            return data
        }
    end

    #MD:
    #: ByteTranslate(byte) - translate the single byte given to the equivalent
    #: UTF-8 codepoint raw data value.
    #:
    method ByteTranslate(byte)
        local   temp                    #LV: temporary to string value of byte

        IsOn()
        DebugPrint("byte:", ::ximage(byte))

        if ::type(byte) == ("string" | "cset") then {
            temp := ::string(byte)
            if *byte = 1 then {
                DebugPrint("byte:", ::ximage(byte), " __ByteTranslate[",
                           ::ximage(byte), "]:", ::ximage(__ByteTranslate[byte]))

                return __ByteTranslate[byte]
            } else if *byte > 1 then {
                DebugPrint("byte[1]:", ::ximage(byte[1]), " __ByteTranslate[",
                           ::ximage(byte[1]), "]:", ::ximage(__ByteTranslate[byte[1]]))

                return __ByteTranslate[byte[1]]
            } else {
                fail
            }
        } else {
            fail
        }
    end


    #### Completed ####
    #MD:
    #: Multibyte(codepoint) - tests that the supplied value is a single UTF-8
    #: codepoint and then checks if that codepoint consists of more than one
    #: byte character. If it is, the value is returned as a UTF-8 string,
    #: otherwise it fails. The value is coerced into a UTF-8 string, a runtime
    #: abort is performed with a value of 103 - "not a string"
    #:
    #:@param codepoint  a string of one or more bytes which will be tested to see
    #:@                 if it is a valid multi-byte UTF-8 codepoint
    #:@returns          the standard UFT-8 object that represents the codepoint
    #:@                 value supplied, if it is a single codepoint of 2 or
    #:@                 more bytes.
    #:@fails            if there are more than 1 codepoints or there is a single
    #:@                 byte.
    #:
    method Multibyte(codepoint)
        local   local_codepoint         #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /codepoint & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                                  __AuxMessage(103))

        #
        # as a matter of course, we coerce the parameter into a UTF-8 object, if
        # the value cannot be coerced, it will cause a runtime abort with
        # 103 - not a string error code
        #
        local_codepoint := __Coerce(codepoint)

        DebugPrint("coerced parameter codepoint:", ::ximage(local_codepoint.String()))

        #
        # We want to ensure that there is only a single codepoint to be tested.
        # We do this as a precaution to ensure that any time we are testing for
        # multi-byte codepoints, we will be returning only a single codepoint.
        # If the codepoint is invalid, Size will also fail and the procedure
        # will fail.
        #
        if Size(local_codepoint) = 1 & *local_codepoint.String() > 1 then {
            #
            # Since this is found to be a valid UTF-8, test if it is the start of
            # a multi-byte codepoint and return codepoint or fail
            #
            return SpecialString(local_codepoint.String())

        #
        # otherwise we will fail on multiple codepoints or invalid codepoints
        #
        } else {
            fail
        }
    end

    #MD:
    #: SpecialString(str) - this will return the specially defined UTF-8 object
    #: representing this string. If this is used appropriately, only one object will
    #: ever be created as the UTF-8 representation of this string.
    #:
    #: If the str is not a valid UTF-8 sequence, it will fail.
    #:
    method SpecialString(str)
        local   temp                    #LV: temporary to hold raw data

        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(103))

        (::type(str) == "string" & \__SpecialUTF8Strings[str] & return .__SpecialUTF8Strings[str])

        if temp := Valid(str) & /__SpecialUTF8Strings[str] then {
            return .(__SpecialUTF8Strings[temp] := UTF8::UTF8.New(temp))
        }
    end

    ############################################################################
    ############################################################################
    #                                                                          #
    #   Translation methods to/from Unicode 32 representations                  #
    #                                                                          #
    #       ToUnicode32(utf8)                                                  #
    #       Unicode32To(U32rep)                                                #
    #                                                                          #
    ############################################################################
    ############################################################################


    #MD:
    #: ToUnicode32(ustr) - a method to turn a UTF-8 string into a sequence of
    #: Unicode string format "U+FFFFFFFF" values. It will suspend on each codepoint.
    #: If the UTF-8 string supplied has invalid codepoints, it will fail instead
    #: of producing any values. If the supplied value cannot be coerced into a UTF-8
    #: value, runtime abort with a 103 - not a string error.
    #:
    #: This will convert a supplied UTF-8 string into the equivalent sequence of
    #: Unicode 32 bit hex strings. (unicon strings)
    #:
    #:@param ustr       this is a string value to be converted
    #:@suspends         a ASCII string representing the hex value of each codepoint
    #:@                 found in the UTF-8 string supplied
    method ToUnicode32(ustr)
        local   byte,                   #LV: temporary to hold the current byte
                                        #:   being looked at as the first byte
                                        #:   of a possible multi-byte codepoint

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                bytecnt,                #LV: the byte count expected based on the
                                        #:   first byte being checked

                i,                      #LV: a temporary used for moving through
                                        #:   the bytes of a multi-byte codepoint

                val,                    #LV: accumulated bit-string representing
                                        #:   the eventual 32 bit number to be
                                        #:   converted to a hex string value

                valo,                   #LV: the hex-string value obtained

                data,                   #LV: the selected codepoint from the
                                        #:   string supplied
                bytechar,               #LV: the character value of the byte found
                bytestr                 #LV: the raw data for the codepoint

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # We need to coerce the supplied value to a UTF-8 string first. The coerced value
        # is placed in a new variable to ensure that the original value/variable
        # supplied is not altered. If cannot be coerced then a runtime abort
        # is performed with a value of 103 - "not a string"
        #
        local_ustr := __Coerce(ustr)

        #
        # for each codepoint in the supplied string, we need the integer value
        # of the first byte as well as the codepoint itself. If the string has
        # any invalid codepoints, it will fail immediately and produce no values.
        #
        if Valid(local_ustr) then {
            every byte := ::ord(bytechar := (bytestr:= ForEach(local_ustr).String())[1]) do {
                #
                # the first byte of any UTF-8 codepoint gives an indicator of how
                # many bytes are in that codepoint. this count is found by a lookup
                # table that was initialised on class creation.
                #
                bytecnt := __UTF8SizeTable[bytechar]

                #
                # using the byte count, we want to select out only those bits which
                # are actually significant to the 32 bit value. See the discussion
                # above of how the encoding of UTF-8 codepoints are done as compared
                # to the 32 bit encodings of UTF32 codepoints.
                #
                case bytecnt of {
                    #
                    # for a single byte encoding, the last 7 bits are significant
                    # as the actual value of the character
                    1 : {
                        val := ::iand(16r7f, byte)
                    }

                    #
                    # for a 2 byte encoding, the last 5 bits are significant for the
                    # actual value of the character
                    #
                    2 : {
                        val := ::iand(16r1f, byte)
                    }

                    #
                    # for a 3 byte encoding, it is the last four bits that are
                    # significant to the actual value of the character
                    #
                    3 : {
                        val := ::iand(16rf, byte)
                    }

                    #
                    # for a 4 byte encoding, it is the last 3 bits that are
                    # significant for the actual value of the character
                    #
                    4 : {
                        val := ::iand(16r7, byte)
                    }

                    #
                    # this needs further investigation as to whether or not we should
                    # allow leading 0 bytes. In the Unicode documents there was some
                    # reference to this, but I am rethinking this case and just
                    # failing at this point or not covering this case, or even making
                    # it an error. still deciding
                    #
                    default : {
                        val := 0
                    }
                }

                #
                # for each additional byte of a multi-byte codepoint, we need to
                # select out the significant bits from each byte and append this to
                # the collected bits as a number (integer value). This requires that
                # we shift the collected bits by 6 bits and add in the last 6 bits
                # from each additional byte.
                #
                every i := 2 to bytecnt do {
                    byte := ::ord(bytestr[i])
                    if ::iand(16rc0, byte) = 16r80 then {
                        val := ::ishift(val, 6) + ::iand(16r3f, byte)
                    }
                }

                #
                # we now use the IPL procedure to convert the collected bits into a
                # hex-string
                #
                valo := ::hexstring(val,(bytecnt < 4 | 5))

                #
                # the final phase is the prepend the Unicode string designator to the
                # hex-string and then suspend it back to the calling expression.
                #
                suspend "U+" || valo
            }
        }
        #
        # There are two reasons for getting to this point in the code
        #   1). The UTF-8 string object contains invalid codepoints
        #   2). There are no more codepoints to process in the UTF-8 string
        # In both cases, we just want to fail at this point.
        #
    end

    #### Completed ####
    #MD:
    #: Unicode32To(unicode) - given an string representing a Unicode 32 codepoint
    #: in the format U+(hex digits), this method will create a single UTF-8
    #: codepoint. If unicode represents a codepoint greater than the largest
    #: possible UTF-8 codepoint, the method will fail. If num is not a
    #: valid integer number, the method will fail.
    #:
    #:@param unicode    string value representing a UTF-8 codepoint
    #:@returns          UTF-8 string object represented by the hexadecimal number
    #:@fails            if integer is not a valid integer number, if the
    #:@                 integer number is > 16r10FFFF, if the value supplied
    #:@                 cannot be converted to a unicon/icon integer
    #:
    method Unicode32To(unicode)
        local   local_unicode           #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /unicode & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                                __AuxMessage(103, 4))

        #
        # Since we have a method that will convert a hexadecimal code string to
        # the relevant UTF-8 string object. All we need do, is to strip off the
        # prefix value "U+" and use the resulting hexadecimal string value as
        # the parameter to the HexTo method to do the work for us
        #
        if local_unicode := ::string(\unicode) then {
            local_unicode := ::map(local_unicode, &lcase, &ucase)
            local_unicode ? {
                ="U+" &
                return HexTo(::tab(0))
            }
            #
            # if the required prefix is not there, we fail
            #
        }
        #
        # otherwise we fail
        #
    end

    ############################################################################
    ############################################################################
    #                                                                          #
    #   Translation of a hex/decimal number representations to the appropriate #
    #   UTF-8 codepoint.                                                       #
    #                                                                          #
    #       HexTo(hexval)                                                      #
    #       DecimalTo(number)                                                  #
    #                                                                          #
    ############################################################################
    ############################################################################

    #### Completed ####
    #MD:
    #: HexTo(hexcode) - given a string representing a hexadecimal number, this
    #: method will create a single UTF-8 codepoint. If hexcode represents a
    #: number greater than the largest possible UTF-8 codepoint, the method will
    #: fail. If hexcode is not a valid hexadecimal number, the method will fail.
    #:
    #:@param hexcode    string value representing a hexadecimal number
    #:@returns          UTF-8 string object represented by the hexadecimal number
    #:@fails            if hexcode is not a valid hexadecimal number, if the
    #:@                 hexadecimal number is > 10FFFF, if the values supplied
    #:@                 cannot be converted to a unicon/icon string
    #:
    method HexTo(hexcode)
        local   val,                    #LV: temporary to hold the integer
                                        #:   equivalent of the supplied hex
                                        #:   value

                local_hexcode,          #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                valo,                   #LV: temporary to hold the string made
                                        #:   up of the calculated bytes

                c1,                     #LV: temporary to hold the first byte

                c2,                     #LV: temporary to hold the second byte,
                                        #:   if it exists

                c3,                     #LV: temporary to hold the third byte,
                                        #:   if it exists

                c4                      #LV: temporary to hold the fourth byte,
                                        #:   if it exists

        /hexcode & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                                __AuxMessage(103, 4))

        #
        # hexcode must be a string or a value convertible to a string
        #
        if local_hexcode := ::string(\hexcode) then {
            #
            # for consistency purposes, convert all uppercase to lowercase letters
            #
            local_hexcode := ::map(local_hexcode, &lcase, &ucase)

            #
            # the value supplied must be a hexadecimal representation, so we use
            # the IPL hexcvt library procedure "hex" to do the conversion to the
            # relevant integer value.
            #
            if val := ::hex(local_hexcode) then {
                #
                # there are four cases we need to test for with the hexadecimal
                # number. It will represent a 1, 2, 3 or byte codepoint.
                #
                # 1 byte is represented by values between 0 and 7F
                # 2 bytes are represented by values between 80 and 7FF
                # 3 bytes are represented by values between 800 and FFFF
                # 4 bytes are represented by values between 10000 and 10FFFF
                #
                # we check for the 1 byte case (0 - 7F)
                #
                if val < 16r80 then {
                    #
                    # In this particular case, the call to iand is actually not
                    # needed, as the value will be in the single 7 bit range.
                    # However, to keep consistency with the code in each of the
                    # following branches, we make the call to iand. This specifies
                    # precisely what we need in terms of our algorithm.
                    #
                    # Use the unicon function iand to select out the low 7 bits
                    # of the number supplied and use this as the number for the
                    # first (and only) character. The first bit is 0 and the
                    # next 7 bits hold the significant bits of the codepoint.
                    #
                    #   0xxxxxxx        - iand with     0000000001111111    (16r7f)
                    #
                    # to select out the required low 7 bits
                    #
                    c1 := ::iand(16r7f, val)

                    #
                    # we now use this to get the required byte value which will
                    # be used later to create a UTF-8 string object.
                    #
                    valo := ::char(c1)

                #
                # we check for the two byte case (80 - 7FF)
                #
                } else if val < 16r800 then {
                    #
                    # For the 2 byte case, the significant bits that represent
                    # the codepoint in the first byte are the low 5 bits and in
                    # the second byte are the low 6 bits.
                    #
                    # Hence, in the first byte we have
                    #
                    #   xxxxxyyyyyy     - iand with     0000011111000000    (16r7c0)
                    #
                    # to select out the high 5 bits needed for the first byte
                    # and then shift these bits to the right by 6 bits
                    #
                    #   xxxxx           - ior with      0000000011000000    (16rc0)
                    #
                    # to get the first byte
                    #
                    #   110xxxxx
                    #
                    # to select out the bits for the second byte, we need to
                    # select out the low 6 bits.
                    #
                    # Hence, in the second byte we have
                    #
                    #   xxxxxyyyyyy     - iand with     0000000000111111    (16r3f)
                    #
                    # to select out the low 6 bits needed for the second byte
                    # and then ior these bits with
                    #
                    #   yyyyyy          - ior with      0000000010000000    (16r80)
                    #
                    # to get the second byte
                    #
                    #   10yyyyyy
                    #
                    c1 := ::ior(16rc0, ::ishift(::iand(16r7c0, val), -6))
                    c2 := ::ior(16r80, ::iand(16r3f, val))

                    #
                    # the codepoint is then the concatenation of the first byte
                    # with the second byte. We use char() to convert each
                    # integer byte value into the corresponding character
                    #
                    valo := ::char(c1) || ::char(c2)

                #
                # we check for the three byte case
                #
                } else if val < 16r10000 then {
                    #
                    # For the 3 byte case, the significant bits that represent
                    # the codepoint in the first byte are the low 4 bits and in
                    # the second and third bytes are the low 6 bits in each.
                    #
                    # Hence, in the first byte we have
                    #
                    #   xxxxyyyyyyzzzzzz
                    #       - iand with     000000001111000000000000    (16rf000)
                    #
                    # to select out the high 4 bits needed for the first byte
                    # and then shift these bits to the right by 12 bits
                    #
                    #   xxxx
                    #       - ior with      000000000000000011100000    (16re0)
                    #
                    # to get the first byte
                    #
                    #   1110xxxx
                    #
                    # to select out the bits for the second byte, we need to
                    # select out the middle 6 bits.
                    #
                    # Hence, in the second byte we have
                    #
                    #   xxxxyyyyyyzzzzzz
                    #       - iand with     000000000000111111000000    (16rfc0)
                    #
                    # to select out the middle 6 bits needed for the second byte
                    # and then shift these bits to the right by 6 bits
                    #
                    #   yyyyyy
                    #       - ior with      000000000000000010000000    (16r80)
                    #
                    # to get the second byte
                    #
                    #   10yyyyyy
                    #
                    # to select out the bits for the third byte, we need to
                    # select out the low 6 bits.
                    #
                    # Hence, in the second byte we have
                    #
                    #   xxxxyyyyyyzzzzzz
                    #       - iand with     000000000000000000111111    (16r3f)
                    #
                    # to select out the low 6 bits needed for the third byte
                    # and then ior these bits with
                    #
                    #   zzzzzz
                    #       - ior with      000000000000000010000000    (16r80)
                    #
                    # to get the third byte
                    #
                    #   10zzzzzz
                    #
                    c1 := ::ior(16re0, ::ishift(::iand(16rf000, val), -12))
                    c2 := ::ior(16r80, ::ishift(::iand(16rfc0, val), -6))
                    c3 := ::ior(16r80, ::iand(16r3f, val))

                    #
                    # the codepoint is then the concatenation of the first,
                    # second and third bytes. We use char() to convert each
                    # integer byte value into the corresponding character
                    #
                    valo := ::char(c1) || ::char(c2) || ::char(c3)

                #
                # we check for the four byte case
                #
                } else if val < 16r200000 then {
                    #
                    # For the 4 byte case, the significant bits that represent
                    # the codepoint in the first byte are the low 3 bits and in
                    # the second , third and fourth bytes are the low 6 bits in each.
                    #
                    # Hence, in the first byte we have
                    #
                    #   xxxyyyyyyzzzzzzwwwwww
                    #       - iand with     000111000000000000000000    (16r1c0000)
                    #
                    # to select out the high 4 bits needed for the first byte
                    # and then shift these bits to the right by 18 bits
                    #
                    #   xxx
                    #       - ior with      000000000000000011110000    (16rf0)
                    #
                    # to get the first byte
                    #
                    #   11110xxx
                    #
                    # to select out the bits for the second byte, we need to
                    # select out the upper middle 6 bits.
                    #
                    # Hence, in the second byte we have
                    #
                    #   xxxxyyyyyyzzzzzzwwwwww
                    #       - iand with     000000111111000000000000    (16r3f000)
                    #
                    # to select out the middle 6 bits needed for the second byte
                    # and then shift these bits to the right by 6 bits
                    #
                    #   yyyyyy
                    #       - ior with      000000000000000010000000    (16r80)
                    #
                    # to get the second byte
                    #
                    #   10yyyyyy
                    #
                    # to select out the bits for the third byte, we need to
                    # select out the low 6 bits.
                    #
                    # Hence, in the third byte we have
                    #
                    #   xxxxyyyyyyzzzzzzwwwwww
                    #       - iand with     000000000000111111000000    (16rfc0)
                    #
                    # to select out the low 6 bits needed for the third byte
                    # and then ior these bits with
                    #
                    #   zzzzzz
                    #       - ior with      000000000000000010000000    (16r80)
                    #
                    # to get the third byte
                    #
                    #   10zzzzzz
                    #
                    # to select out the bits for the fourth byte, we need to
                    # select out the low 6 bits.
                    #
                    # Hence, in the fourth byte we have
                    #
                    #   xxxxyyyyyyzzzzzzwwwwww
                    #       - iand with     000000000000000000111111    (16r3f)
                    #
                    # to select out the low 6 bits needed for the fourth byte
                    # and then ior these bits with
                    #
                    #   wwwwww
                    #       - ior with      000000000000000010000000    (16r80)
                    #
                    # to get the third byte
                    #
                    #   10wwwwww
                    #
                    c1 := ::ior(16rf0, ::ishift(::iand(16r1c0000, val), -18))
                    c2 := ::ior(16r80, ::ishift(::iand(16r3f000, val), -12))
                    c3 := ::ior(16r80, ::ishift(::iand(16rfc0, val), -6))
                    c4 := ::ior(16r80, ::iand(16r3f, val))

                    #
                    # the codepoint is then the concatenation of the first,
                    # second, third and fourth bytes. We use char() to convert each
                    # integer byte value into the corresponding character
                    #
                    valo := ::char(c1) || ::char(c2) || ::char(c3) || ::char(c4)

                #
                # otherwise the hexadecimal number supplied is too large and so we
                # fail
                #
                } else {
                    fail
                }
                #
                # create a new UTF-8 string object based on the sequences of bytes
                # that have been created above.
                #
                return SpecialString(valo)
            }
        }
        #
        # otherwise we fail
        #
    end

    #### Completed ####
    #MD:
    #: DecimalTo(num) - given an integer representing a UTF-8 codepoint, this
    #: method will create a single UTF-8 codepoint. If num represents a
    #: number greater than the largest possible UTF-8 codepoint, the method will
    #: fail. If num is not a valid integer number, the method will fail.
    #:
    #:@param num        integer value representing a UTF-8 codepoint
    #:@returns          UTF-8 string object represented by the hexadecimal number
    #:@fails            if integer is not a valid integer number, if the
    #:@                 integer number is > 16r10FFFF, if the value supplied
    #:@                 cannot be converted to a unicon/icon integer
    #:
    method DecimalTo(num)
        /num & AbortMessage(102, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(102))

        #
        # Since we have a method that will convert a hexadecimal code string to
        # the relevant UTF-8 string object. All we need do, is to convert the
        # supplied integer value to a hexadecimal string value and use the HexTo
        # method to do the work for us
        #
        return HexTo(hexstring(::integer(\num)))
    end

    #MD:
    #: AmpPos() - provides an analog to &pos for the current position in the
    #: UTF8 value in terms of codepoints.
    #:
    method AmpPos()
        local   j := 1,                 #LV: we initialise this to the start
                                        #:   position in the string relative
                                        #:   to the LHS

                k := 1                  #LV:


        #
        #
        # To determine the current codepoint position based off &pos, we need to step through each
        # codepoint. We have available to us a table which has each valid byte
        # that can be found in any UTF-8 codepoint and which has a count of 1
        # for each starting byte of a codepoint and has a count of 0 for each
        # subsequent byte of a multi-byte codepoint.
        #
        # In effect, we use the table to determine how many codepoints we have
        # seen and we count the actual number of bytes we have passed.
        # so k count of bytes, j count of codepoints
        #

        DebugPrint("&subject:", image(&subject))

        while k < &pos do {
            DebugPrint("&subject[k]:", image(&subject[k]), "\n",
                       "__UTF8SizeTable[&subject[k]]:", __UTF8SizeTable[&subject[k]], "\n",
                       "k:", k)

            k +:= __UTF8SizeTable[&subject[k]]
            j +:= 1

            DebugPrint("&pos:", &pos, "\n", "j:", j, "\n", "k:", k)

        }

        DebugPrint("&pos:", &pos, "\n", "j:", j, "\n", "k:", k)

        return .j
    end

    ###########################################################################
    ###########################################################################
    #
    # The following methods are related to the operator overload - method
    # functionality. The generic specification of the methods is defined in the
    # super class ClassObject.
    #
    # They can be used as method calls or via the operator overload facilities
    # that can be switch on with the build directive in define.h of
    # #define OVLD 1
    #
    # The relevant operators and methods mapping for this class and associated
    # objects are:
    #
    #   Method                  Overload Method                     Operator
    #
    #   Equiv                   __eqv__                             ===     binary
    #   Nequiv                  __neqv__                            ~===    binary
    #   LexEqual                __lexeq__                           ==      binary
    #   LexGTorEq               __lexge__                           >>=     binary
    #   LexGT                   __lexgt__                           >>      binary
    #   LexLTorEq               __lexle__                           <<=     binary
    #   LexLT                   __lexlt__                           <<      binary
    #   LexNE                   __lexne__                           ~==     binary
    #   Concatenate             __cat__                             ||      binary
    #   ForEach                 __bang__                            !       unary
    #   Size                    __size__                            *       unary
    #   Random                  __random__                          ?       unary
    #   Subsection              __sect__                            [:]     ternary
    #   Subscript               __subsc__                           []      binary
    #   TabMatch                __tabmat__                          =       unary
    #
    ###########################################################################

    ############## Need to check semantics of Equiv against === #############
    #MD:
    #: Equiv(val1, val2) - test if the two values are UTF-8 strings and that they
    #: have the same underlying values. Both are coerced to UTF-8 strings first
    #: if required. If they are not equivalent, then fail. If they are equivalent,
    #: the second value is returned. Equivalence is based on the comparing the
    #: raw data values of each of the UTF-8 objects.
    #:
    #:@param val1           a UTF-8 string or convertible to a UTF-8 string
    #:@param val2           a UTF-8 string or convertible to a UTF-8 string
    #:@returns              val2 if it is equivalent to val1 as UTF-8 strings
    #:@aborts               if either string cannot be converted to an UTF-8 string
    #:@fails                if val2 is not equivalent to val1 as UTF-8 strings
    #:
    method Equiv(val1, val2)
        local   local_val1,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_val2             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /val1 & /val2 & return val2
        /val1 & fail
        /val2 & fail
        ::image(val1) == ::image(val2) & return val2
        #
        # we coerce the supplied valid first. If cannot be coerced
        # then aruntime abort is performed with a value of 103 - "not a string"
        #

        DebugPrint("val1:", ::image(val1))

        local_val1 := __Coerce(val1)

        DebugPrint("local_val1:", ::image(local_val1))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #

        DebugPrint("val2:", ::image(val2))

        local_val2 := __Coerce(val2)

        DebugPrint("local_val2:", ::image(local_val2))

        return (local_val1.String() == local_val2.String()) & val2
    end

    #MD:
    #: Nequiv(val1, val2) - test if the two values are UTF-8 strings and that they
    #: don't have the same underlying values. Both are coerced to UTF-8 strings first
    #: if required. If they are not equivalent, then fail. If they are equivalent,
    #: the second value is returned.
    #:
    #:@param val1           a UTF-8 string or convertible to a UTF-8 string
    #:@param val2           a UTF-8 string or convertible to a UTF-8 string
    #:@returns              val2 if it is equivalent to val1 as UTF-8 strings
    #:@aborts               if either string cannot be converted to an UTF-8 string
    #:@fails                if val2 is not equivalent to val1 as UTF-8 strings
    #:
    method Nequiv(val1, val2)
        return (not Equiv(val1, val2)) & val2
    end

    #MD:
    #:
    #:
    method LexEquals(str1, str2)
        local   local_str1,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_str2              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /str2 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str1 := __Coerce(str1)

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str2 := __Coerce(str2)

        #
        # We now compare the UFT-8 string data of both supplied values and see
        # if they are lexically equal using the string operator ==.
        #
        # It should be noted that this is compares by codepoint and not by lexeme.
        # There are much more complex rules relating to the comparison of Unicode
        # lexemes which this code does not get involved in.
        #
        # If such comparisons are required then additional procedures or methods
        # will need to be written on a case by case basis. This will typically
        # involve understanding specific language (human written forms) requirements.
        #
        return (local_str1.String() == local_str2.String()) & str2
    end

    #MD:
    #:
    #:
    method LexGTorEq(str1, str2)
        local   local_str1,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_str2              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /str2 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str1 := __Coerce(str1)

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str2 := __Coerce(str2)

        #
        # We now compare the UFT-8 string data of both supplied values and see
        # if they are lexically equal using the string operator >>=.
        #
        # It should be noted that this only compares by codepoint and not by lexeme.
        # There are much more complex rules relating to the comparison of Unicode
        # lexemes which this code does not get involved in.
        #
        # If such comparisons are required then additional procedures or methods
        # will need to be written on a case by case basis. This will typically
        # involve understanding specific language (human written forms) requirements.
        #
        return (local_str1.String() >>= local_str2.String()) & str2
    end

    #MD:
    #:
    #:
    method LexGT(str1, str2)
        local   local_str1,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_str2              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /str2 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str1 := __Coerce(str1)

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str2 := __Coerce(str2)

        #
        # We now compare the UFT-8 string data of both supplied values and see
        # if they are lexically equal using the string operator >>.
        #
        # It should be noted that this is compares by codepoint and not by lexeme.
        # There are much more complex rules relating to the comparison of Unicode
        # lexemes which this code does not get involved in.
        #
        # If such comparisons are required then additional procedures or methods
        # will need to be written on a case by case basis. This will typically
        # involve understanding specific language (human written forms) requirements.
        #
        return (local_str1.String() >> local_str2.String()) & str2
    end

    #MD:
    #:
    #:
    method LexLTorEq(str1, str2)
        local   local_str1,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_str2              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /str2 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str1 := __Coerce(str1)

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str2 := __Coerce(str2)

        #
        # We now compare the UFT-8 string data of both supplied values and see
        # if they are lexically equal using the string operator <<=.
        #
        # It should be noted that this is compares by codepoint and not by lexeme.
        # There are much more complex rules relating to the comparison of Unicode
        # lexemes which this code does not get involved in.
        #
        # If such comparisons are required then additional procedures or methods
        # will need to be written on a case by case basis. This will typically
        # involve understanding specific language (human written forms) requirements.
        #
        return (local_str1.String() <<= local_str2.String()) & str2
    end

    #MD:
    #:
    #:
    method LexLT(str1, str2)
        local   local_str1,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_str2              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /str2 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str1 := __Coerce(str1)

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str2 := __Coerce(str2)

        #
        # We now compare the UFT-8 string data of both supplied values and see
        # if they are lexically equal using the string operator <<.
        #
        # It should be noted that this is compares by codepoint and not by lexeme.
        # There are much more complex rules relating to the comparison of Unicode
        # lexemes which this code does not get involved in.
        #
        # If such comparisons are required then additional procedures or methods
        # will need to be written on a case by case basis. This will typically
        # involve understanding specific language (human written forms) requirements.
        #
        return (local_str1.String() << local_str2.String()) & str2
    end

    #MD:
    #:
    #:
    method LexNE(str1, str2)
        local   local_str1,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_str2              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /str2 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str1 := __Coerce(str1)

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str2 := __Coerce(str2)

        #
        # We now compare the UFT-8 string data of both supplied values and see
        # if they are lexically equal using the string operator ~==.
        #
        # It should be noted that this is compares by codepoint and not by lexeme.
        # There are much more complex rules relating to the comparison of Unicode
        # lexemes which this code does not get involved in.
        #
        # If such comparisons are required then additional procedures or methods
        # will need to be written on a case by case basis. This will typically
        # involve understanding specific language (human written forms) requirements.
        #
        return (local_str1.String() ~== local_str2.String()) & str2
    end

    #### Completed ####
    #MD:
    #: Concatenate(ustr1, ustr2) - This method will concatenate the two UTF-8
    #: strings together and return a new UTF-8 string object. If either of the
    #: two values are not UTF-8 strings, they will be coerced if possible.
    #:
    #:@param ustr1          this will be the start of the new UTF-8 string
    #:@param ustr2          this will be the end of the new UTF-8 string
    #:@returns              the concatenation of both strings
    #:@aborts               if either string cannot be coerced into a UTF-8
    #:@                     string
    #:
    method Concatenate(ustr1, ustr2)
        local   str,                    #LV: temporary to hold the result of
                                        #:   the concatenation of the data values
                                        #:   of each of the source UTF-8 strings

                local_ustr1,            #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_ustr2             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /ustr1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(103))

        /ustr2 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(103))

        #
        # we coerce the supplied ustr1 first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #

        DebugPrint("ustr1:", ::image(ustr1))

        local_ustr1 := __Coerce(ustr1)

        DebugPrint("local_ustr1:", ::image(local_ustr1))

        #
        # we coerce the supplied ustr2 second. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #

        DebugPrint("ustr2:", ::image(ustr2))

        local_ustr2 := __Coerce(ustr2)

        DebugPrint("local_ustr2:", ::image(local_ustr2))

        str := local_ustr1.String() || local_ustr2.String()

        DebugPrint("str:", ::image(str))

        return UTF8::UTF8.New(str)
    end

    #MD:
    #: Complement(val)
    #:
    method Complement(val)
        /val & AbortMessage(104, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(104))

        return UTF8::UTF8Set.Complement(val)
    end

    #MD:
    #: Difference(val1, val2)
    #:
    method Difference(val1, val2)
        /val1 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(120))

        /val2 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(120))

        return UTF8::UTF8Set.Difference(val1, val2)
    end

    #MD:
    #: Intersect(val1, val2)
    #:
    method Intersect(val1, val2)
        /val1 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(120))

        /val2 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(120))

        return UTF8::UTF8Set.Intersect(val1, val2)
    end

    #MD:
    #: Union(val1, val2)
    #:
    method Union(val1, val2)
        /val1 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(120))

        /val2 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(120))

        return UTF8::UTF8Set.Union(val1, val2)
    end

    #### Completed ####
    #MD:
    #: ForEach(str) - a procedure that returns the individual UTF-8 codepoints (as a
    #: unicon string value) found in the specified string. The value given is
    #: coerced into a UTF-8 string. If the value cannot be coerced into a UTF-8
    #: string, a runtime abort is performed with a value of 103 - "not a string"
    #:
    #: We may the starting assumption that each string supplied is made up of
    #: valid UTF-8 codepoints. Using the first byte as a indicator of the UTF-8
    #  codepoint size, we collect that number of bytes together and check them
    #; to see if they represent a valid UTF-8 codepoint. If that test fails, we
    #: will return the UTF-8 Invalid Codepoint codepoint instead. This allows
    #; any code using this method to determine for itself what action to take
    #: when an invalid codepoint is found in the source strings.
    #:
    #:@param str            a value that we will coerce to a UTF-8 object, if it
    #:@                     cannot so be coerced, runtime abort with a 103 error
    #:@suspends             each UTF-8 codepoint or the UTF-8 Invalid Codepoint
    #:@                     codepoint if it cannot resolve the byte string as a
    #:@                     valid UTF-8 codepoint
    #:@fails                when there are no more to be had
    #:
    method ForEach(str)
        local   codepoint,              #LV: temporary storage for each codepoint
                                        #:   found

                local_str,              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.
                tmp_val                 #LV: temporary value holder

        /str & AbortMessage(116, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(103))

        DebugPrint("type of str: ", ::type(str), " value of str: ", ::image(str))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str := __Coerce(str)

        DebugPrint("type of local_str: ", ::type(local_str), " value of local_str: ", ::image(local_str))

        #
        # scan the raw data and select out each of the UTF-8 codepoints (1 to 4 bytes)
        #
        local_str.String() ? {
            #
            # while we still have bytes to analyse, not yet reached the end of
            # the supplied data, we extract each codepoint. The general assumption
            # here is that we will find valid codepoints. If 1 or more invalid
            # bytes are found as starting bytes, we will return the UTF-8 Invalid
            # Codepoint, which is a valid UTF-8 codepoint, and is supposed to be
            # returned if we find a invalid byte sequence.
            #
            # We may need to look at another method that will allow analysis of
            # the byte stream so that local handling of invalid bytes can be done
            # in a user defined systematic manner.
            #
            while not ::pos(0) do {
                #
                # We collect the first byte as this will be used for determining
                # what further bytes are to be collected.
                #
                codepoint := ::move(1)

                DebugPrint("codepoint:", ::ximage(codepoint))

                #
                # we will now collect what further bytes are required based on
                # the size that is expected for the first byte.
                #
                codepoint ||:= ::move(1 <= (__UTF8SizeTable[.codepoint] - 1))

                DebugPrint("codepoint:", ::ximage(codepoint))

                #
                # we check if this codepoint is valid or not and suspend it or
                # the to Invalid codepoint to the calling expression
                #
                tmp_val := (SpecialString(Valid(codepoint)) | __UTF8InvalidChar)

                DebugPrint("tmp_val:", ::image(tmp_val))

                suspend tmp_val
            }
        }
    end

    #### Completed ####
    #MD:
    #: Size(value) - procedure calculates the actual number of UTF-8 codepoints found in
    #: the supplied value. The value supplied is coerced to a UTF-8 string
    #: first. This is the method that will be called if the * operator is used on
    #: any UTF-8 object. This assumes that only valid UTF-8 codepoints will exist
    #: in the supplied value.
    #:
    #: Need to look at what changes are required to validate the value to ensure
    #: that only valid UTF-8 codepoints are found. It may require the coercion
    #: functionality be responsible for this, but it may also need to be done in
    #: other places as well. Need to think about this carefully as non-UTF-8 byte
    #: strings will contain non valid codepoints and this is allowable.
    #:
    #:@param str        any value, though usually a unicon string or a UTF-8
    #:@                 object
    #:@returns          a count of the UTF-8 codepoints found in the source object
    #:@aborts           if a value cannot be coerced to a UTF-8 object (103 - not a string)
    #
    method Size(str)
        local   cnt                     #LV: temporary to count the number of
                                        #:   codepoints in the parameter

        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                                         __AuxMessage(103))

        #
        # in the case of raw strings, we will run through and treat it as a
        # potential UTF-8 sequence of code points
        #
        if ::type(str) == "string" then {
            cnt := 0
            every cnt +:= __UTF8CountTable[!str]
            return cnt
        #
        # if it is already an UTF-8 string or a UTF-8 set, then just return the
        # size of the object.
        #
        } else if IsObject(str, "UTF8", "UTF8Set") then {
            return str.Size()
        #
        # if is is the &null value then the size will be 0
        } else if /str then {
            return 0
        #
        # for anything else, including Unicon csets, just return the size as given
        # by the * operator
        #
        } else {
            return *str
        }
    end

    #MD:
    #:
    #:
    method Random(str)
        local   local_str,              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                i                       #LV: temporary to hold the random position
                                        #:   that will be chosen from within the
                                        #:   coerced UTF-8 string data.

        /str & AbortMessage(113, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str := __Coerce(str)

        #
        # we choose a random integer value between 1 and the length of the UTF-8 value
        # value
        #
        i := ?local_str.Size()

        #
        # We return the codepoint found at that position
        #
        return __GetAt(local_str, i)
    end

    #### Completed ####
    #MD:
    #: Subsection(ustr, i, j) - this will return a new UTF-8 object that contains
    #: the UTF-8 codepoints between the i'th and j'th positions in the UTF-8 value.
    #: If only the first position is specified, it will return a single codepoint
    #: as the new UTF-8 value. If only the second is specified, it will return all
    #: codepoints between the first and the specified position as a new UTF-8 value.
    #: If neither are specified, the first codepoint will be returned. The same
    #: semantic rules that apply to Unicon strings apply to these values, if i < j,
    #: they will be swapped, if either is negative or 0, then the position is
    #: measured from the RHS and converted to the actual position as measured
    #: from the LHS of the UTF-8 value.
    #:
    #:@param ustr           a value that can be coerced into a UTF-8 value
    #:@param i              the starting position of the substring (measured by
    #:@                     codepoints)
    #:@param j              the ending position of the substring (measured in
    #:@                     codepoints)
    #:@fails                if either of the positions are outside of the actual
    #:@                     size of the UTF-8 value (measured in codepoints)
    #:@aborts               if the supplied value cannot be converted to an UTF-8
    #:@                     value
    #:
    method Subsection(ustr, i, j)
        local   tmpstr                  #LV: temporary to hold coerced string
        #
        # we coerce the supplied ustr first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        # Then we use the internal method to select the i'th to j'th codepoints
        # as a new UTF-8 object
        #
        /ustr & AbortMessage(110, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(110))

        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          __AuxMessage(101))

        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        /j & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          __AuxMessage(101))

        (j := ::integer(j)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 4), ::image(j))

        tmpstr := __Coerce(ustr)

        DebugPrint("tmpstr:", ::image(tmpstr), " i:", i, " j:", j)

        return __GetAt(tmpstr, i, j)
    end

    #MD:
    #: Subscript(str, i) - select the i'th codepoint from the supplied UTF-8
    #: value. This uses the Subsection method to obtain the relevant codepoint.
    #: The same position semantics apply to Subscript as for Subsection
    #:
    #:@param ustr           a value that can be coerced into a UTF-8 value
    #:@param i              the starting position of the substring (measured by
    #:@                     codepoints)
    #:@fails                if either of the positions are outside of the actual
    #:@                     size of the UTF-8 value (measured in codepoints)
    #:@aborts               if the supplied value cannot be converted to an UTF-8
    #:@                     value
    #:
    method Subscript(str, i)
        #
        # As Subsection will select only a single codepoint when supplied the
        # start position, we do not have to supply the second parameter to the
        # method call.
        #
        /str & AbortMessage(114, keyword("&file", , 1), keyword("&line", , 1), "\noffending value: &null\nMissing UTF8 Object/value")

        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          __AuxMessage(101))

        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        return Subsection(str, i, i + 1)
    end

    #### Completed ####
    #MD:
    #: TabMatch(str) - In the current scanning environment, is the supplied string
    #: the match for the current position, if it is, return the value as a UTF-8
    #: string, otherwise fail
    #:
    #:@param str            a value that can be coerced into a UTF-8 string for
    #:@                     matching at the current scanning position
    #:@returns              the UTF-8 coerced value if it matches
    #:@fails                if it doesn't match
    #:@aborts               if the value cannot be converted to a UTF-8 string
    #:
    method TabMatch(str)
        local   local_str               #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                                         __AuxMessage(103))

        #
        # we coerce the supplied valid first. If cannot be coerced
        # then a runtime abort is performed with a value of 103 - "not a string"
        #
        local_str := __Coerce(str)
        return =local_str.String() & local_str
    end

    ###########################################################################
    #
    # These methods are design to match the unicon built-in functions applicable
    # to strings.
    #
    #   Method                  Built-in Function       Returns(?)/Suspends(*)
    #
    #   Any                     any                     integer?
    #   Bal                     bal                     integer*
    #   Center                  center                  UTF-8 string object
    #   Char                    char                    UTF-8 string object
    #   Cset                    cset                    UTF-8 set object
    #   Detab                   detab                   UTF-8 string object
    #   Entab                   entab                   UTF-8 string object
    #   Find                    find                    integer*
    #   Left                    left                    UTF-8 string object
    #   Many                    many                    integer?
    #   Map                     map                     UTF-8 string object
    #   Match                   match                   integer?
    #   Move                    move                    UTF-8 string object
    #   Ord                     ord                     integer
    #   Pos                     pos                     integer?
    #   Repl                    repl                    UTF-8 string object
    #   Reverse                 reverse                 UTF-8 string object
    #   Right                   right                   UTF-8 string object
    #   String                  string                  UTF-8 string object
    #   Tab                     tab                     UTF-8 string object
    #   Trim                    trim                    UTF-8 string object
    #   Upto                    upto                    integer*



    #MD:
    #: Any(uset, ustr, i, j) - this is the procedure for UTF-8 strings that matches
    #: the standard unicon/icon function any(c,s,i,j). When used as a generator,
    #: it will generate each position in ustr just before any character found in
    #: uset between the positions i and j in the string, Like upto(), this will use
    #: &subject as the source string. The normal rules apply if either of i or j are
    #: less or equal to 0 (set the position from the RHS of the UTF-8 string
    #: object).
    #:
    #:@param uset       must be specified, this will be coerced to a UTF-8 cset and
    #:@                 these codepoints. If this cset is not specified and error
    #:@                 104 - not a cset will be raised.
    #:@param ustr       the string to be scanned, which will be coerced to UTF-8 if necessary
    #:@                 if necessary, if not specified, the current &subject will be
    #:@                 used and this will be coerced into a UTF-8 string.
    #:@param i          this specifies the starting position in the UTF-8 string,
    #:@                 if not specified, this will be set to 1
    #:@param j          this specifies then end position of the scan, if not specified
    #:@                 this will be specified to the position after the last UTF-8
    #:@                 codepoint.
    #:@suspends         each position in the string just before any character found
    #:@                 in the specified UTF-8 cset
    #:
    method Any(uset, ustr, i, j)
        local   c,                      #LV:

                found,                  #LV:

                fc,                     #LV:

                local_uset,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                sz                      #LV:

        /uset & AbortMessage(104, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(104))

        #
        # test that uset is not null, if it is procedure fails
        #
        /uset & fail

        #
        # coerce the supplied value to a UFT-8 cset
        #
        local_uset := Cset(uset)

        #
        #######################################################################
        # common set-up code for string analysis code
        #
        # if i is not specified, it will be set to either the current value of &pos
        # if the string to scan is null or 1 (the start of the string to scan)
        #
        /i := ((/ustr & &pos) | 1)

        #
        # we will use the current value found in &subject if no string is specified
        # for scanning
        #
        /ustr := &subject

        #
        # we need to coerce the given string value to a UTF-8 string if it needs this
        #
        local_ustr := __Coerce(ustr)

        #
        # we need the size of the UTF-8 string for various calculations, so get it now
        #
        sz := local_ustr.Size()
        #
        # if j is not supplied, we set it to a position just after the last codepoint
        #
        /j := sz + 1
        #
        #
        # convert i and j to integers if necessary and abort if either cannot be
        # converted to an integer
        #
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        (j := ::integer(j)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 4), ::image(j))

        #
        # here we do the calculations to convert i and j from negative values to
        # the proper positions values. negative values measure from the RHS of the
        # string and we need to start from the LHS of the string
        #
        (i <= 0) & i := sz + i + 1
        (j <= 0) & j := sz + j + 1

        #
        # there is nothing to do if the values of i and j are equal, so the procedure
        # just needs to fail as we will have no results
        #
        (i ~= j) | fail

        #
        # it is allowed that i can be greater than j, in this case we simply swap the
        # values before starting the scan of the string
        #
        (i > j) & i :=: j

        #
        # the last check we need to make is that j is in fact less than or equal
        # the length of the string + 1. if it is greater than the last possible
        # position for a string, we then need to fail.
        #
        (j <= sz + 1) | fail

        #
        # end of common set-up code for string analysis code
        #######################################################################
        #

        DebugPrint("i:", i, "\n", "j:", j, "\n", "sz:", sz)

        fc := __GetAt(local_ustr, i)

        DebugPrint("fc.String():", ::ximage(fc.String()))

        return local_uset.Member(fc) & (i + 1)
    end

    #MD:
    #: Bal(uset, usetopen, usetclose, ustr, i, j) - this is the procedure for
    #: UTF-8 strings that matches the standard unicon/icon function bal(c,c,c,s,i,j).
    #: Loop through characters in ustr[i:j].  When a character in usetopen
    #: is found, increment cnt; when a character in usetclose is found, decrement
    #: cnt.  When cnt is 0 there have been an equal number of occurrences
    #: of characters in usetopen and usetclose, i.e., the string to the left of
    #: i is balanced.  If the string is balanced and the current character
    #: (ustr[i]) is in uset, suspend with i.  Note that if cnt drops below
    #:  zero, bal fails.
    #:
    #:@param uset       The set of UTF-8 codepoints to skip
    method Bal(uset, usetopen, usetclose, ustr, i, j)
        local   cnt := 0,               #LV:

                testchar,               #LV:

                local_uset,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_usetopen,         #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_usetclose,        #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                sz                      #LV:

        /uset & fail
        #
        #######################################################################
        # common set-up code for string analysis code
        #
        # if i is not specified, it will be set to either the current value of &pos
        # if the string to scan is null or 1 (the start of the string to scan)
        #
        /i := ((/ustr & &pos) | 1)

        #
        # we will use the current value found in &subject if no string is specified
        # for scanning
        #
        /ustr := &subject

        #
        # we need to coerce the given string value to a UTF-8 string if it needs this
        #
        local_ustr := __Coerce(ustr)

        #
        # we need the size of the UTF-8 string for various calculations, so get it now
        #
        sz := local_ustr.Size()

        #
        # if j is not supplied, we set it to a position just after the last codepoint
        #
        /j := sz + 1

        #
        #
        # convert i and j to integers if necessary and abort if either cannot be
        # converted to an integer
        #
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        (j := ::integer(j)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 4), ::image(j))

        #
        # here we do the calculations to convert i and j from negative values to
        # the proper positions values. negative values measure from the RHS of the
        # string and we need to start from the LHS of the string
        #
        (i <= 0) & i := sz + i + 1
        (j <= 0) & j := sz + j + 1

        #
        # there is nothing to do if the values of i and j are equal, so the procedure
        # just needs to fail as we will have no results
        #
        (i ~= j) | fail

        #
        # it is allowed that i can be greater than j, in this case we simply swap the
        # values before starting the scan of the string
        #
        (i > j) & i :=: j
        #
        # the last check we need to make is that j is in fact less than or equal
        # the length of the string + 1. if it is greater than the last possible
        # position for a string, we then need to fail.
        #
        (j <= sz + 1) | fail

        #
        # end of common set-up code for string analysis code
        #######################################################################

        DebugPrint("i:", i, "\n", "j:", j, "\n", "sz:", sz, "\n",
                   "coerced parameter ustr.String():", ::ximage(local_ustr.String()))
        #
        # Ensure that the supplied value is convertible to a UTF-8 cset. In the
        # case of using the default of all possible codpoints, the indicator is
        # that value supplied is &null. Implementation-wise, since csets are atomic
        # unchangeable values, there is only one UTF-8 cset that exists covering
        # all codepoints. There are a number of other UTF-8 csets that are similarly
        # created and are stored as values in the UTF8Set class.
        #
        local_uset := Cset(uset)

        #
        # if no open UTF-8 cset is supplied, we want the default cset to be '('
        # and make sure it is a UTF-8 cset
        #
        /usetopen := '('
        local_usetopen := Cset(usetopen)

        #
        # if no close UTF-8 cset is supplied, we want the default cset to be ')'
        # and make sure it is a UTF-8 cset
        #
        /usetclose := ')'
        local_usetclose := Cset(usetclose)

        DebugPrint("coerced parameter usetopen.UTF8Value().String():",
                   ::ximage(local_usetopen.UTF8Value().String()), "\n",
                   "coerced parameter usetclose.UTF8Value().String():", ::ximage(local_usetclose.UTF8Value().String()), "\n",
                   "coerced parameter ustr.String():", ::ximage(local_ustr.String()))

        #
        # Since this is a UTF-8 string object, we cannot use the string scanning
        # operation due to the variable size codepoints. So, we will simply step
        # through the UTF-8 string object, codepoint by codepoint.
        #
        while i <= j do {
            #
            # Loop through characters in s[i:j].
            # When cnt is 0 there have been an equal number of occurrences
            # of characters in c2 and c3, i.e., the string to the left of
            # i is balanced.  If the string is balanced and the current character
            # (s[i]) is in c, suspend with i.
            #
            # When a character in c2 is found, increment cnt; when a character
            # in c3 is found, decrement cnt.
            #
            # Note that if cnt drops below zero, Bal fails.
            #
            # We need the codepoint at the current position for testing against
            # each of the supplied UTF-8 csets. This position will be specified
            # by i as this is how we will move through the UTF-8 string object
            #
            testchar := __GetAt(local_ustr, i)

            DebugPrint("testchar.String():", ::ximage(testchar.String()))

            #
            #
            if cnt = 0 & local_uset.Member(testchar) then {
                suspend i
            }

            if local_usetopen.Member(testchar) then {
                cnt +:= 1
            } else if local_usetclose.Member(testchar) then {
                cnt -:= 1
            }

            if cnt < 0 then {
                fail
            }
            i +:= 1
        }
        fail
    end

    #
    # Center - a procedure to centre a UTF-8 string in a field of size i with
    # padding provided by a second UTF-8 string.
    #
    # Center(ustr, i:1, pstr:" ")
    # produce a UTF-8 string of i UTF-8 characters. If i > Size(ustr) then ustr
    # is padded equally on the left and right with pstr to length i. There are
    # proviso's with this padding, if the number of padding characters is odd
    # then the extra padding character is placed to the right of ustr. In
    # addition, if the Size(pstr) > 1 then the left and right padding values are
    # created as follows:
    #
    #   Left padding -  replicate the padding string enough times to get a string
    #                   long enough to cover at least the left padding count.
    #       pstr : "xyz" require 5 padding characters
    #       lpad : "xyzxyz" take the 5 left most characters and use as padding
    #       apad : "xyzyy"
    #
    #   Right padding - replicate the padding string enough times to get a string
    #                   long enough to cover at least the right padding count.
    #       pstr : "xyz" require 5 padding characters
    #       rpad : "xyzxyz" take the 5 right most characters and use as padding
    #       apad : "yzxyz"
    #
    #   Using the above example of a padding string, we obtain the following
    #   variations based on the count for padding
    #
    #   Padding count       Left padding value      Right padding value
    #
    #       1                       ""                      "z"
    #       2                       "x"                     "z"
    #       3                       "x"                     "yz"
    #       4                       "xy"                    "yz"
    #       5                       "xy"                    "xyz"
    #       6                       "xyz"                   "xyz"
    #       7                       "xyz"                   "zxyz"
    #       8                       "xyzx"                  "zxyz"
    #       9                       "xyzx"                  "yzxyz"
    #      10                       "xyzxy"                 "yzxyz"
    #      11                       "xyzxy"                 "xyzxyz"
    #      12                       "xyzxyz"                "xyzxyz"
    #
    # When i < Size(ustr), there are no padding characters applied to the
    # string. The string returned is a sub-string consisting of the i centre
    # characters of ustr. However, the specific sub-string returned is created
    # as follows:
    #
    #   For strings with an odd size: example uses 7 characters "abcdefg"
    #       Since Unicon/Icon treats character position as existing between
    #       characters, the following diagram shows how the characters are
    #       selected.
    #
    #
    #       1   2   3   4   5   6   7   8
    #      -7  -6  -5  -4  -3  -2  -1   0
    #       |   |   |   |   |   |   |   |
    #         a   b   c   d   e   f   g
    #
    #   Results:
    #       i           sub-string selected
    #       1                   "d"
    #       2                   "de"
    #       3                   "cde"
    #       4                   "cdef"
    #       5                   "bcdef"
    #       6                   "bcdefg"
    #       7                   "abcdefg"
    #
    #   Formulae to select start and end position of sub-string:
    #       start pos :     j := (Size(ustr) + 3 - i) / 2
    #       end pos :       j + i
    #
    #   For strings with an even size: example uses 8 characters "abcdefgh"
    #
    #
    #       1   2   3   4   5   6   7   8   9
    #      -8  -7  -6  -5  -4  -3  -2  -1   0
    #       |   |   |   |   |   |   |   |   |
    #         a   b   c   d   e   f   g   h
    #
    #   Results:
    #       i           sub-string selected
    #       1                   "e"
    #       2                   "de"
    #       3                   "def"
    #       4                   "cdef"
    #       5                   "cdefg"
    #       6                   "bcdefg"
    #       7                   "bcdefgh"
    #       8                   "abcdefgh"
    #
    #   Formulae to select start and end position of sub-string:
    #       start pos :     j := ((Size(ustr) + 2) / 2) - (i / 2)
    #       end pos :       j + i
    #

    method Center(ustr, i, pstr)
        local   data,                   #LV:

                result,                 #LV:

                j,                      #LV:

                k,                      #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_pstr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                lstr,                   #LV:

                rstr,                   #LV:

                s,                      #LV:

                sz                      #LV:

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # set the default field size to 1 if not supplied
        #
        /i := 1

        #
        # set the default padding character/codepoint to the default of ascii
        # space
        #
        /pstr := " "

        #
        # we need to coerce the given string value to a UTF-8 string if it needs this
        #
        local_ustr := __Coerce(ustr)

        #
        # we need to coerce the padding string value to a UTF-8 string if it needs this
        #
        local_pstr := __Coerce(pstr)

        DebugPrint("coerced parameter ustr.String():", ::ximage(local_ustr.String()), "\n",
                   "coerced parameter pstr.String():", ::ximage(local_pstr.String()), "\n",
                   "i:", i)

        #
        # we need the i value to be convertible to an integer and the value to
        # be greater or equal to 0
        #
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))


        (i >= 0) | AbortMessage(205, keyword("&file", , 1), keyword("&line", , 1), "\noffending value for i: ", i)

        #
        # we also need to have the size of the supplied string for further calculations
        # related to padding and field sized
        #
        sz := local_ustr.Size()

        DebugPrint("sz:", sz)

        #
        # there are three conditions that can arise:
        #   (1). the size of the field is greater than the size of the display string
        #   (2). the size of the field is even
        if i > sz then {
            j := (i - sz) / 2
            k := (i - sz - j)

            DebugPrint("j:", j, "\n", "k:", k)

            lstr := __GetAt(Repl(local_pstr, j), 1, j + 1)
            rstr := Repl(local_pstr, k)
            s := rstr.Size()
            rstr := __GetAt(rstr, s - k + 1, 0)
            data := local_ustr.String()

            DebugPrint("lstr.String():", ::ximage(lstr.String()), "\n",
                       "rstr.String():", ::ximage(rstr.String()), "\n",
                       "s:", s, "\n",
                       "rstr.String():", ::ximage(rstr.String()), "\n",
                       "data:", ::ximage(data))

            return UTF8::UTF8.New(lstr.String() || data || rstr.String())

        } else if i % 2 = 0 then {
            k := i / 2
            j := ((sz + 2) / 2) - k
            data := __GetAt(local_ustr, j, j + i)
            return data

        } else {
            j := (sz + 3 - i) / 2
            data := __GetAt(local_pstr, j, j + i)
            return data
        }
    end

    method Char(i)
        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          __AuxMessage(101))

        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        return DecimalTo(i)
    end

    method Cset(val)
        /val & fail
        return UTF8Set.__Coerce(val)
    end

    #
    # In the method Detab, each of the UTF-8 codepoints is treated as a single
    # width character. The list of tabs will default to a list of size one with
    # a single value of 9. The default tab-stop size is 8. If there are no
    # more values to be found in the tab-stop list, the tab-stop columns will be
    # calculated at distance of "interval" characters after the last calculated
    # tab-stop. "interval" is the default tab-stop size.
    #
    # The backspace character will cause the current column to be decreased by 1,
    # except when the column is in the start position of 1.
    #
    # Newline and Carriage Return characters will cause the column count to be reset
    # to 1.
    #
    # Tab characters will add enough spaces from the current column to the next
    # tab-stop position.
    #
    # If no change is made to the supplied UTF-8 string, it is returned, otherwise,
    # the new UTF-8 string is returned.
    #
    method Detab(ustr, i[])
        local   j := 1,                 #LV:

                k,                      #LV:

                str := "",              #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.
                isorted,                #LV: sorted list of i values
                ch,                     #LV:

                col := 1,               #LV:

                tst,                    #LV:

                interval := 8           #LV:

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        (*i = 0) & i := [9]
        isorted := ::sort(i)
        if not ::equiv(i, isorted) then {
            j := i[1]
            every k := i[2 to *i] do {
                if j < k then {
                    j := k
                } else {
                    break
                }
            }
            AbortMessage(210, keyword("&file", , 1), keyword("&line", , 1), "\noffending value: ", k)
        }

        DebugPrint("ximage(i):", ::ximage(i), "\n", "interval:", interval)

        local_ustr := __Coerce(ustr)

        DebugPrint("coerced parameter ustr.String():", ::ximage(local_ustr.String()))

        every tst := ::type(!i) do {
            (tst == "integer") | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                              __AuxMessage(101, 3), ::image(i))
        }

        DebugPrint("all elements of i are integers")

        every (tst := !i) do {
            (j <=:= tst) | AbortMessage(210, keyword("&file", , 1), keyword("&line", , 1), "\noffending values for i: ", i, " and j:", j)
        }

        DebugPrint("all elements of i are in sequence")

        j := 1
        k := i[j]

        DebugPrint("j:", j, "\n", "k:", k)

        every ch := local_ustr.ForEach().String() do {
            DebugPrint("ch:", ::ximage(ch), "\n","col:", col)

            case ch of {
                "\b" : {
                    str ||:= ch
                    col -:= (col > 1)
                }
                "\n" | "\r" : {
                    str ||:= ch
                    col := j := 1
                }
                "\t" : {
                    every col to k - 1 do {
                        str ||:= " "
                    }
                    col := k
                    j +:= 1
                    k := (i[j] | (k + interval))
                }
                default : {
                    str ||:= ch
                    col +:= 1
                    j +:= (col = k > 1)
                }
            }

            DebugPrint("col:", col, "\n", "k:", k)

            if col = k then {
                j +:= 1
                k := (i[j] | k + interval)
            }

            DebugPrint("str:", ::ximage(str))

        }

        DebugPrint("str:", ::ximage(str), "\n",
                   "coerced parameter ustr.String():", ::ximage(local_ustr.String()))

        return if local_ustr.String() == str then {
            local_ustr
        } else {
            UTF8::UTF8.New(str)
        }
    end

    method Entab(ustr, i[])
        local   j := 1,                 #LV:

                k,                      #LV:

                str := "",              #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.
                isorted,                #LV: sorted list of i values
                ch,                     #LV:

                col := 1,               #LV:

                p,                      #LV:

                tst,                    #LV:

                interval := 8           #LV:

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        (*i = 0) & i := [9]
        isorted := ::sort(i)
        if not ::equiv(i, isorted) then {
            j := i[1]
            every k := i[2 to *i] do {
                if j < k then {
                    j := k
                } else {
                    break
                }
            }
            AbortMessage(210, keyword("&file", , 1), keyword("&line", , 1), "\noffending value: ", k)
        }

        DebugPrint("ximage(i):", ::ximage(i), "\n", "interval:", interval)

        local_ustr := __Coerce(ustr)

        DebugPrint("coerced parameter ustr.String():", ::ximage(local_ustr.String()))

        every tst := ::type(!i) do {
            (tst == "integer") | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                              __AuxMessage(101, 3), ::image(i))
        }

        DebugPrint("all elements of i are integers")

        every (tst := !i) do {
            (j <=:= tst) | AbortMessage(210, keyword("&file", , 1), keyword("&line", , 1), "\noffending values for i: ", i, " and j:", j)
        }

        DebugPrint("all elements of i are in sequence")

        j := 1
        k := i[j]

        DebugPrint("j:", j, "\n", "k:", k)

        every ch := local_ustr.ForEach().String() do {

            DebugPrint("ch:", ::ximage(ch), "\n", "col:", col)

            case ch of {
                "\b" : {
                    str ||:= ch
                    col -:= (col > 1)
                }
                "\n" | "\r" : {
                    str ||:= ch
                    col := j := 1
                }
                "\t" : {
                    col := k
                    j +:= 1
                    k := (i[j] | (k + interval))
                }
                " " : {
                    str ||:= " "
                    col +:= 1

                    DebugPrint("col:", col, "\n", "k:", k)

                    if col = k then {

                        DebugPrint("str:", ::ximage(str))

                        str := ::reverse(str)

                        DebugPrint("str:", ::ximage(str))

                        if p := ::many(' ', str) then {
                            str := "\t" || str[p:0]

                            DebugPrint("str:", ::ximage(str))

                        }

                        DebugPrint("str:", ::ximage(str))

                        str := ::reverse(str)

                        DebugPrint("str:", ::ximage(str))
                    }
                }
                default : {
                    str ||:= ch
                    col +:= 1
                    j +:= (col = k > 1)
                }
            }

            DebugPrint("col:", col, "\n", "k:", k)

            if col = k then {
                j +:= 1
                k := i[j] | k + interval
            }

            DebugPrint("str:", ::ximage(str))

        }

        DebugPrint("str:", ::ximage(str), "\n",
                   "coerced parameter ustr.String():", ::ximage(local_ustr.String()))

        return if local_ustr.String() == str then {
            local_ustr
        } else {
            UTF8::UTF8.New(str)
        }
    end

    method Find(ustr1, ustr, i, j)
        local   sz1,                    #LV:

                termcnt,                #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_ustr1,            #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                str3,                   #LV:

                str4,                   #LV:

                sz                      #LV:

        /ustr1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(103))

        #
        #######################################################################
        # common set-up code for string analysis code
        #
        # if i is not specified, it will be set to either the current value of &pos
        # if the string to scan is null or 1 (the start of the string to scan)
        #
        /i := ((/ustr & &pos) | 1)
        #
        # we will use the current value found in &subject if no string is specified
        # for scanning
        #
        /ustr := &subject
        #
        # we need to coerce the given string value to a UTF-8 string if it needs this
        #
        local_ustr := __Coerce(ustr)
        #
        # we need the size of the UTF-8 string for various calculations, so get it now
        #
        sz := local_ustr.Size()
        #
        # if j is not supplied, we set it to a position just after the last codepoint
        #
        /j := sz + 1
        #
        #
        # convert i and j to integers if necessary and abort if either cannot be
        # converted to an integer
        #
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))
        (j := ::integer(j)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 4), ::image(j))
        #
        # here we do the calculations to convert i and j from negative values to
        # the proper positions values. negative values measure from the RHS of the
        # string and we need to start from the LHS of the string
        #
        (i <= 0) & i := sz + i + 1
        (j <= 0) & j := sz + j + 1
        #
        # there is nothing to do if the values of i and j are equal, so the procedure
        # just needs to fail as we will have no results
        #
        (i ~= j) | fail
        #
        # it is allowed that i can be greater than j, in this case we simply swap the
        # values before starting the scan of the string
        #
        (i > j) & i :=: j
        #
        # the last check we need to make is that j is in fact less than or equal
        # the length of the string + 1. if it is greater than the last possible
        # position for a string, we then need to fail.
        #
        (j <= sz + 1) | fail
        #
        # end of common set-up code for string analysis code
        #######################################################################

        DebugPrint("coerced parameter ustr.String():", ::ximage(local_ustr.String()), "\n",
                   "i:", i, "\n", "j:", j, "\n", "sz:", sz)

        local_ustr1 := __Coerce(ustr1)

        DebugPrint("coerced parameter ustr1.String():", ::ximage(local_ustr1.String()))

        sz1 := local_ustr1.Size()

        DebugPrint("sz1:", sz1)

        termcnt := j - sz1

        DebugPrint("termcnt:", termcnt)

        str3 := local_ustr1.String()
        while i <= termcnt do {
            str4 := __GetAt(local_ustr, i, i + sz1).String()

            DebugPrint("str3:", ::ximage(str3), "\n", "str4:", ::ximage(str4), "\n", "i:", i)

            if str3 == str4 then {

                DebugPrint("suspend ", i)

                suspend i
            }
            i +:= 1
        }
        fail
    end

    method Left(ustr, i, pstr)
        local   local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_pstr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                j,                      #LV:

                k,                      #LV:

                rstr,                   #LV:

                s                       #LV:

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /i := 1
        (::type(i) == "integer") | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                                __AuxMessage(101, 5), ::type(i))

        (i > 0) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                               __AuxMessage(101, 6), i)

        /pstr := " "
        local_ustr := __Coerce(ustr)
        local_pstr := __Coerce(pstr)

        DebugPrint("coerced parameter ustr.String():", ::ximage(local_ustr.String()), "\n",
                   "coerced parameter pstr.String():", ::ximage(local_pstr.String()), "\n",
                   "i:", i)

        j := local_ustr.Size()

        DebugPrint("j:", j)

        if i > j then {
            k := (i - j)
            rstr := Repl(local_pstr, k)
            s := rstr.Size()

            DebugPrint("k:", k, "\n", "rstr.String():",
                       ::ximage(rstr.String()), "\n", "s:", s)

            rstr := __GetAt(rstr, s - k + 1, 0)

            DebugPrint("rstr.String():", ::ximage(rstr.String()), "\n",
                       "coerced parameter ustr.String():", ::ximage(local_ustr.String()))

            return UTF8::UTF8.New(local_ustr.String() || rstr.String())
        } else {
            return __GetAt(local_ustr, 1, i)
        }
    end

    method Many(uset, ustr, i, j)
        local   c,                      #LV:

                found,                  #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_uset,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                fc,                     #LV:

                startpos,               #LV:

                sz                      #LV:

        /uset & AbortMessage(104, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(104))

        #
        #######################################################################
        # common set-up code for string analysis code
        #
        # if i is not specified, it will be set to either the current value of &pos
        # if the string to scan is null or 1 (the start of the string to scan)
        #
        /i := ((/ustr & &pos) | 1)
        #
        # we will use the current value found in &subject if no string is specified
        # for scanning
        #
        /ustr := &subject
        #
        # we need to coerce the given string value to a UTF-8 string if it needs this
        #
        local_ustr := __Coerce(ustr)
        #
        # we need the size of the UTF-8 string for various calculations, so get it now
        #
        sz := local_ustr.Size()
        #
        # if j is not supplied, we set it to a position just after the last codepoint
        #
        /j := sz + 1
        #
        #
        # convert i and j to integers if necessary and abort if either cannot be
        # converted to an integer
        #
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        (j := ::integer(j)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 4), ::image(j))

        #
        # here we do the calculations to convert i and j from negative values to
        # the proper positions values. negative values measure from the RHS of the
        # string and we need to start from the LHS of the string
        #
        (i <= 0) & i := sz + i + 1
        (j <= 0) & j := sz + j + 1
        #
        # there is nothing to do if the values of i and j are equal, so the procedure
        # just needs to fail as we will have no results
        #
        (i ~= j) | fail
        #
        # it is allowed that i can be greater than j, in this case we simply swap the
        # values before starting the scan of the string
        #
        (i > j) & i :=: j
        #
        # the last check we need to make is that j is in fact less than or equal
        # the length of the string + 1. if it is greater than the last possible
        # position for a string, we then need to fail.
        #
        (j <= sz + 1) | fail
        #
        # end of common set-up code for string analysis code
        #######################################################################
        #
        local_uset := Cset(uset)
        startpos := i
        while i < j & local_uset.Member(__GetAt(local_ustr, i)) do {
            i +:= 1
        }
        if i = startpos then {
            fail
        } else {
            return i
        }
    end

    method Map(ustr, ustr1, ustr2)
        local   i,                      #LV:

                ch,                     #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_ustr1,            #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_ustr2,            #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                k,                      #LV:

                str := ""               #LV:

        static  maptab,                 #SV:

                istr1,                  #SV:

                istr2                   #SV:

        initial {
            maptab := ::table()
        }

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        local_ustr := __Coerce(ustr)
        /ustr1 := ::string(&ucase)
        local_ustr1 := __Coerce(ustr1)

        DebugPrint("coerced parameter ustr1.String():", ::ximage(local_ustr1.String()))

        /ustr2 := ::string(&lcase)
        local_ustr2 := __Coerce(ustr2)
        (*ustr1 ~== *ustr2) & AbortMessage(208, keyword("&file", , 1), keyword("&line", , 1))

        DebugPrint("coerced parameter ustr2.String():", ::ximage(local_ustr2.String()))

        ((i := local_ustr1.Size()) = local_ustr2.Size()) |
                AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103, 7), "(",
                             local_ustr1.String(), ") and (", local_ustr2.String(), ")")

        if /istr1 | /istr2 | istr1.String() ~== local_ustr1.String() | istr2.String() ~== local_ustr2.String() then {
            # recompute mapping table
            every k := 1 to i do {
                maptab[__GetAt(local_ustr1,k).String()] := __GetAt(local_ustr2, k).String()
            }
            # set static value to current values
            istr1 := local_ustr1
            istr2 := local_ustr2
        }

        DebugPrint("ximage(maptab):", ::ximage(maptab))

        every ch := ForEach(local_ustr).String() do {
            str ||:= (\maptab[ch] | ch)
        }

        DebugPrint("str:", ::ximage(str))

        return if local_ustr.String() == str then {
            local_ustr
        } else {
            UTF8::UTF8.New(str)
        }
    end

    method Match(ustr1, ustr, i, j)
        local   sz1,                    #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_ustr1,            #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                str,                    #LV:

                str3,                   #LV:

                sz                      #LV:

        /ustr1 & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(103))

        #
        #######################################################################
        # common set-up code for string analysis code
        #
        # if i is not specified, it will be set to either the current value of &pos
        # if the string to scan is null or 1 (the start of the string to scan)
        #
        /i := ((/ustr & &pos) | 1)
        #
        # we will use the current value found in &subject if no string is specified
        # for scanning
        #
        /ustr := &subject
        #
        # we need to coerce the given string value to a UTF-8 string if it needs this
        #
        local_ustr := __Coerce(ustr)
        #
        # we need the size of the UTF-8 string for various calculations, so get it now
        #
        sz := local_ustr.Size()
        #
        # if j is not supplied, we set it to a position just after the last codepoint
        #
        /j := sz + 1
        #
        #
        # convert i and j to integers if necessary and abort if either cannot be
        # converted to an integer
        #
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        (j := ::integer(j)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 4), ::image(j))

        #
        # here we do the calculations to convert i and j from negative values to
        # the proper positions values. negative values measure from the RHS of the
        # string and we need to start from the LHS of the string
        #
        (i <= 0) & i := sz + i + 1
        (j <= 0) & j := sz + j + 1
        #
        # there is nothing to do if the values of i and j are equal, so the procedure
        # just needs to fail as we will have no results
        #
        (i ~= j) | fail
        #
        # it is allowed that i can be greater than j, in this case we simply swap the
        # values before starting the scan of the string
        #
        (i > j) & i :=: j
        #
        # the last check we need to make is that j is in fact less than or equal
        # the length of the string + 1. if it is greater than the last possible
        # position for a string, we then need to fail.
        #
        (j <= sz + 1) | fail
        #
        # end of common set-up code for string analysis code
        #######################################################################
        #
        local_ustr1 := __Coerce(ustr1)
        sz1 := local_ustr1.Size()
        sz1 <= (j - i) | fail
        str3 := local_ustr1.String()
        str := __GetAt(local_ustr, i, i + sz1).String()
        if str == str3 then {
            return i + sz1
        } else {
            fail
        }
    end

    method Move(i)
        local   ustr,                   #LV:
                newpos,                 #LV:
                newcpos,                #LV:
                curpos,                 #LV:
                curcpos,                #LV:
                sz                      #LV:

        ustr := __Coerce(&subject)
        sz := ustr.Size()
        curpos := &pos
        curcpos := AmpPos()
        /i := 1

        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        DebugPrint("curpos(&pos):", curpos, "\n", "curcpos:", curcpos, "i:", i, "sz:", sz)

        newcpos := curcpos + i
        (newcpos < 0) & fail
        (newcpos > sz + 1) & fail
        newpos := __ActualPos(ustr, newcpos)

        DebugPrint("curpos:", curpos, " curcpos:", curcpos)
        DebugPrint("newpos:", newpos, " newcpos:", newcpos)
        DebugPrint("&pos:", &pos, "\n", "newpos:", newpos)

        suspend UTF8::UTF8.New(&subject[.&pos:&pos <- newpos])
    end

    method Ord(ustr)
        local   i,                      #LV:

                str,                    #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                n := 0,                 #LV:

                m                       #LV:

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        local_ustr := __Coerce(ustr)
        (local_ustr.Size() = 1) |
            AbortMessage(205, keyword("&file", , 1), keyword("&line", , 1), "\noffending value for Size: ",
                         local_ustr.Size(), " should be of size 1")

        str := ToUnicode32(local_ustr.String())
        str ? {
            ="U+" &
            m := ::tab(0)
            n := ::hex(m)
        }
        return n
    end

    method Pos(i)
        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          __AuxMessage(101))

        integer(i) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                  __AuxMessage(101, 3), ::image(i))

        return pos(__ActualPos(__Coerce(&subject), i))
    end

    method Repl(ustr, i)
        local   data,                   #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                str                     #LV:

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        (i >= 0) | AbortMessage(205, keyword("&file", , 1), keyword("&line", , 1), "\noffending value for i: ", i)

        local_ustr := __Coerce(ustr)
        data := local_ustr.String()

        DebugPrint("data:", ::ximage(data), "\n", "i:", i)

        str := data
        every 2 to i do {
            str ||:= data
        }

        DebugPrint("str:", ::ximage(str))

        return UTF8::UTF8.New(str)
    end

    #
    # Reverse the UTF-8 string
    #
    method Reverse(ustr)
        local   nstr := "",             #LV:

                local_ustr              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        local_ustr := __Coerce(ustr)
        every nstr := ForEach(local_ustr).String() || nstr
        return UTF8::UTF8.New(nstr)
    end

    method Right(ustr, i, pstr)
        local   j,                      #LV:

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_pstr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                lstr                    #LV:

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        /i := 1

        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        (i >= 0) | AbortMessage(205, keyword("&file", , 1), keyword("&line", , 1), "\noffending value for i: ", i)

        /pstr := " "
        local_ustr := __Coerce(ustr)
        local_pstr := __Coerce(pstr)
        j := local_ustr.Size()
        if i > j then {
            j := (i - j)
            lstr := __GetAt(Repl(local_pstr, j), 1, j)
            return UTF8::UTF8.New(lstr.String() || local_ustr.String())
        } else {
            j := (j + 1 - i)
            return __GetAt(local_ustr, j, j + i)
        }
    end

    method String(val)
        /val & return ""
        return __Coerce(val).String()
    end

    method Tab(i)
        local   ustr,                   #LV:
                newpos,                 #LV:
                newcpos,                #LV:
                curpos,                 #LV:
                curcpos,                #LV:
                sz                      #LV:

        ustr := __Coerce(&subject)
        sz := ustr.Size()
        curpos := &pos
        curcpos := AmpPos()
        /i := 0
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        DebugPrint("curpos(&pos):", curpos, "\n", "curcpos:", curcpos, "i:", i, "sz:", sz)

        newcpos :=  i
        newpos := __ActualPos(ustr, newcpos)
        (newcpos > sz + 1) & fail

        DebugPrint("curpos:", curpos, " curcpos:", curcpos,
                   "newpos:", newpos, " newcpos:", newcpos,
                   "&pos:", &pos, "\n", "newpos:", newpos)

        suspend UTF8::UTF8.New(&subject[.&pos:&pos <- newpos])
    end

    #MD:
    #: Trim(ustr, uset, i, allutf8blanks) - this is the procedure for UTF-8 strings that matches
    #: the standard unicon/icon function trim(s,c,i). It has an additional parameter
    #: when non-null, will use the various UTF-8 blanks as well as " ". When
    #: i = -1 (default) it will remove from the RHS of the string. When i = 1,
    #: it will remove from the LHS of the string. When i = 0, it will remove
    #: from both ends of the string.
    #:
    #: This version uses the current UTF-8 space definition as of 23-Sept-2018
    #: and may need to be modified in the future. See UTF8Set class definition
    #:
    #:@param ustr       the string, which will be coerced to UTF-8 if necessary
    #:@                 from which the specified uset of codepoints will be
    #:@                 removed.
    #:@param uset       if specified, this will be coerced to a UTF-8 cset and
    #:@                 these codepoints will be the candidates removed from the
    #:@                 specified end/ends. Default is " ", unless overridden by
    #:@                 the allutf8blanks flag.
    #:@param i          this specifies which end/ends to trim, default is -1 (RHS)
    #:@                 1 (LHS), 0 (both ends)
    #:@returns          the trimmed string
    #:
    method Trim(ustr, uset, i, allutf8blanks)
        local   j,                      #LV: temporary to hold position after the
                                        #:   last codepoint in the ustr, used to
                                        #:   test if rightmost codepoints are in uset

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_uset,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                k                       #LV: temporary to hold position before
                                        #:   the first codepoint, used to test if
                                        #:   leftmost codepoints are in uset

        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(103))

        #
        # if uset is &null and depending on whether we want all UTF-8 space
        # codepoints or just the ASCII space, we set the uset to the appropriate
        # default space set.
        #
        /uset := ((/allutf8blanks & UTF8Set.Space()) | UTF8Set.UTF8Space())

        #
        # as a matter of course, we will coerce the cset of characters, this will
        # have no effect if the default has been set
        #
        local_uset := UTF8Set.__Coerce(uset)

        #
        # we coerce the value given and if it is not convertible we will do a
        # runtime abort with a 103 - not a string error
        #
        local_ustr := __Coerce(ustr)

        DebugPrint("coerced parameter ustr.String():", local_ustr.String())

        #
        # if no value has been supplied for which ends to trim, we will use the
        # default of -1, strip from RHS of string
        #
        /i := -1

        #
        # if the value given for which end to start is not an integer, we need
        # to runtime abort with an error of 101 - not an integer
        #
        (::type(i) == "integer") | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                                __AuxMessage(101, 3), ::image(i))

        #
        # we can now set the temporaries to the position after the last codepoint
        # and the position before the first codepoint
        #
        j := local_ustr.Size() + 1
        k := 1

        DebugPrint("i:", i, "\n", "j:", j, "\n", "k:", k)

        #
        # there are two situations in which we want to test each codepoint from
        # the RHS, when we want to trim only from the RHS or when we want to trim
        # from both ends. In addition, we want the temporary to always point to
        # the position after the last non member codepoint of the trim set.
        #
        # so, the loop conditions for RHS testing are when i <= 0 and the position
        # for testing is to the previous codepoint to where we are currently
        # pointing. We will only decrement that pointer if the relevant codepoint
        # is in the test set.
        #
        while (i <= 0) & local_uset.Member(__GetAt(local_ustr, j - 1)) do {
            j -:= 1

            DebugPrint("  j:", j)
        }

        #
        # in a similar manner, there are two situations in which we want to test
        # each codepoint from the LHS, when we want to trim only from the LHS or
        # when we want to trim from both ends. In addition, we want the temporary
        # to always point to the position before the first non member codepoint
        # of the trim set.
        #
        # so, the loop conditions for LHS testing are when i >= 0 and the position
        # for testing is to the next codepoint to where we are currently
        # pointing. We will only increment that pointer if the relevant codepoint
        # is in the test set.
        #
        while (i >= 0) & local_uset.Member(__GetAt(local_ustr, k)) do {
            k +:= 1

            DebugPrint("  k:", k)
        }

        DebugPrint("j:", j, "\n", "k:", k)

        #
        # using the final start and end points, we will create a new UTF-8 string
        # that has been trimmed of the selected set of codepoints
        #
        return UTF8::UTF8.New(__GetAt(local_ustr, k, j))
    end

    #MD:
    #: Upto(uset, ustr, i, j) - this is the procedure for UTF-8 strings that matches
    #: the standard unicon/icon function upto(c,s,i,j). When used as a generator,
    #: it will generate each position in ustr just before any character found in
    #: uset between the positions i and j in the string. Like upto(), this will use
    #: &subject as the source string. The normal rules apply if either of i or j are
    #: less or equal to 0 (set the position from the RHS of the UTF-8 string
    #: object).
    #:
    #:@param uset       must be specified, this will be coerced to a UTF-8 cset and
    #:@                 these codepoints. If this cset is not specified and error
    #:@                 104 - not a cset will be raised.
    #:@param ustr       the string to be scanned, which will be coerced to UTF-8 if necessary
    #:@                 if necessary, if not specified, the current &subject will be
    #:@                 used and this will be coerced into a UTF-8 string.
    #:@param i          this specifies the starting position in the UTF-8 string,
    #:@                 if not specified, this will be set to 1
    #:@param j          this specifies then end position of the scan, if not specified
    #:@                 this will be specified to the position after the last UTF-8
    #:@                 codepoint.
    #:@suspends         each position in the string just before any character found
    #:@                 in the specified UTF-8 cset
    #:
    method Upto(uset, ustr, i, j)
        local   sz,                     #LV: the size of the string (in codepoints)

                local_ustr,             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                local_uset              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /uset & AbortMessage(104, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(104))

        #
        #######################################################################
        # common set-up code for string analysis code
        #
        # if i is not specified, it will be set to either the current value of &pos
        # if the string to scan is null or 1 (the start of the string to scan)
        #
        /i := ((/ustr & &pos) | 1)

        #
        # we will use the current value found in &subject if no string is specified
        # for scanning
        #
        /ustr := &subject

        #
        # we need to coerce the given string value to a UTF-8 string if it needs this
        #
        local_ustr := __Coerce(ustr)

        #
        # we need the size of the UTF-8 string for various calculations, so get it now
        #
        sz := local_ustr.Size()

        #
        # if j is not supplied, we set it to a position just after the last codepoint
        #
        /j := sz + 1

        #
        #
        # convert i and j to integers if necessary and abort if either cannot be
        # converted to an integer
        #
        (i := ::integer(i)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 3), ::image(i))

        (j := ::integer(j)) | AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                                           __AuxMessage(101, 4), ::image(j))

        #
        # here we do the calculations to convert i and j from negative values to
        # the proper positions values. negative values measure from the RHS of the
        # string and we need to start from the LHS of the string
        #
        (i <= 0) & i := sz + i + 1
        (j <= 0) & j := sz + j + 1

        #
        # there is nothing to do if the values of i and j are equal, so the procedure
        # just needs to fail as we will have no results
        #
        (i ~= j) | fail

        #
        # it is allowed that i can be greater than j, in this case we simply swap the
        # values before starting the scan of the string
        #
        (i > j) & i :=: j

        #
        # the last check we need to make is that j is in fact less than or equal
        # the length of the string + 1. if it is greater than the last possible
        # position for a string, we then need to fail.
        #
        (j <= sz + 1) | fail

        #
        # end of common set-up code for string analysis code
        #######################################################################
        #
        # if the supplied cset if null we need to abort with a 104 - cset expected
        #
        \uset | AbortMessage(104, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(104))

        #
        # now we can make sure that the uset value is a UFT-8 cset
        #
        local_uset := Cset(uset)

        #
        # After all that, we now do the scan checking to see if each codepoint
        # in the string is a match to one in the cset supplied. if it is, we
        # suspend it back to the calling expression.
        #
        while (i < j) do {
            if local_uset.Member(__GetAt(local_ustr, i)) then {
                suspend i
            }
            i +:= 1
        }

        #
        # at the completion of the scan, the procedure will fail as per upto()
        # semantics
        #
    end

#MD:
#: initially()
initially()
    #
    # Since we only want this code to ever be run once, we use the initial {}
    # expression to ensure this happens.
    #
    initial {
        #
        # We need to ensure that the superclasses are executed and this is done
        # with the following call. If they have already been executed once, this
        # will be a null operation.
        #
        Execute(ClassObject::ClassClass)

        #
        # We will test if the global debugging is turned on or off.
        #
        IsOn()

        #
        # This is an example of a class that has an associated class. We need to
        # create the linkage between the two classes. This linkage defines that
        # the method New() will create a new member of the associated class that
        # will be treated as a member of this class
        #
        SetClass(UTF8::__UTF8Object)

        #
        # we now set this class name to be the object itself, so that no more
        # objects of this unicon class are created.
        #
        UTF8::UTF8 := self

        #
        # For the UTF-8 system, we need to initialise all the various data structures
        # that will be needed for processing byte strings as UTF-8 sequences.
        #
        __Initialise()

        ::runtime_error_message(2000, "cannot coerce to UTF8 object/value")

        DebugPrint("Have Finished Initialising the Class UTF8")
    }
end

#CD:
#: class __UTF8Object   this is the class that represents each UTF-8 string. Just
#:                      as normal Unicon/Icon strings are immutable, the UTF-8
#:                      string objects will be immutable (by convention), there
#:                      are no methods that allow updating of any of the class
#:                      variables. The methods that enquire of these class variables
#:                      all dereference the actual values on return.
#:
#: I have considered a possible change in the internal structure to store the
#: actual UTF-8 string value into a list of strings, where each element will
#: a consistent codepoint byte size. As yet, this is only an idea and I will
#: wait to see if there is any requirement to do this.
#:
class __UTF8Object : ClassObject(
    __data,                             #CV: This is the internal variable that
                                        #:   holds the actual string of bytes
                                        #:   that represents the sequence of UTF-8
                                        #:   codepoints.

    __asciionly,                        #CV: This is a flag that is used to indicate
                                        #:   that there is only ASCII data being
                                        #:   held in this object, that is just the
                                        #:   the characters that have byte values
                                        #:   that are between 0 and 127.

    __size,                             #CV: this is the number of codepoints within
                                        #:   the UTF-8 object. It is calculated a
                                        #:   creation time and doesn't change
                                        #:   thereafter

    __source_data                       #CV: this will be non-null only if the
                                        #:   original byte source (as UTF-8) has
                                        #:   invalid bytes in it - can be used
                                        #:   for validation or analysis if
                                        #:   needed
)

    #MD:
    #: Data() - returns the raw data that is associated with the UTF-8 string, this
    #: raw data is in a byte string.
    #:
    #:@returns          the raw byte data associated with this UTF-8 object as a
    #:@                 derefernced string value. This is to make sure that the
    #:@                 cannot be used to update the actual variable.
    #:
    method Data()
        return .__data
    end

    ###########################################################################
    #
    # The following methods are the specific methods that will be used in the
    # operator calls. They will all reference the UTF-8 applicable methods.
    #
    #   ~               Complement
    #   --              Difference
    #   **              Intersect
    #   ++              Union
    #   ===             Equiv
    #   ~===            Nequiv
    #   ==              LexEquals
    #   >>=             LexGTorEq
    #   >>              LexGT
    #   <<=             LexLTorEq
    #   <<              LexLT
    #   ~==             LexNE
    #   !               ForEach
    #   ?               Random
    #   =               TabMatch
    #   *               Size
    #   ||              Concatenate
    #   [:]             Subsection
    #   []              Subscript
    #
    ###########################################################################

    #MD:
    #: Complement()
    #:
    method Complement()
        return UTF8::UTF8Set.Complement(self)
    end

    #MD:
    #: Difference(val)
    #:
    method Difference(val)
        /val & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(120))

        return UTF8::UTF8Set.Difference(self, val)
    end

    #MD:
    #: Intersect(val)
    #:
    method Intersect(val)
        /val & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(120))

        return UTF8::UTF8Set.Intersect(self, val)
    end

    #MD:
    #: Union(val)
    #:
    method Union(val)
        /val & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(120))

        return UTF8::UTF8Set.Union(self, val)
    end

    #MD:
    #:
    #:
    method Equiv(val)
        /val & fail
        return UTF8::UTF8.Equiv(self, val)
    end

    #MD:
    #:
    #:
    method Nequiv(val)
        /val & fail
        return UTF8::UTF8.Nequiv(self, val)
    end

    #MD:
    #:
    #:
    method LexEquals(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(103))

        return UTF8::UTF8.LexEquals(self, str)
    end

    #MD:
    #:
    #:
    method LexGTorEq(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(103))

        return UTF8::UTF8.LexGTorEq(self, str)
    end

    #MD:
    #:
    #:
    method LexGT(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(103))

        return UTF8::UTF8.LexGT(self, str)
    end

    #MD:
    #:
    #:
    method LexLTorEq(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(103))

        return UTF8::UTF8.LexLTorEq(self, str)
    end

    #MD:
    #:
    #:
    method LexLT(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(103))

        return UTF8::UTF8.LexLT(self, str)
    end

    #MD:
    #:
    #:
    method LexNE(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8.__AuxMessage(103))

        return UTF8::UTF8.LexNE(self, str)
    end

    #MD:
    #: ForEach() - this method will return each codepoint in sequence from the
    #: UTF-8 string. If the data is ASCII only (byte values between 0 and 127),
    #: we can use the normal generator process for all such strings. It is only
    #: when we have UTF-8 multi-byte characters that we have to use the more
    #: general process of getting each UTF-8 codepoint.
    #:
    #:@suspends         each codepoint in sequence
    #:@fails            when there no more codepoints to be had
    #:
    method ForEach()
        local   temp,
                temp2

        #
        # ASCII is a special case and we can use the standard unicon/icon
        # processes for generating each character
        #

        DebugPrint("__asciionly:", ::image(__asciionly),"\n__data:", ::image(__data))

        if \__asciionly then {
            suspend temp2 := UTF8::UTF8.SpecialString(temp := !__data) do {
                DebugPrint("each ascii element from __data:", image(temp))
                DebugPrint("temp2:", ::image(temp2), " String():", temp2.String())
            }
        #
        # otherwise, we need to do the more complex multi-byte handling
        #
        } else {
            suspend UTF8::UTF8.ForEach(self)
        }
    end

    #MD:
    #:
    #:
    method Random()
        return UTF8::UTF8.Random(self)
    end

    #MD:
    #:
    #:
    method TabMatch()
        return UTF8::UTF8.TabMatch(self)
    end

    #MD:
    #: Size() - returns the stored size of this UTF-8 string. This is the method
    #: that will be called directly if the * operator is used on
    #: any UTF-8 object.
    #:
    #:@returns          a count of the UTF-8 codepoints found in the source object
    #
    method Size()
        #
        # since the size has been determined at creation time, there should be
        # (in normal use), no way that the data has been changed and so the number
        # of codepoints will remain the same for the entire lifetime of this
        # object. Therefore, we will be able to just return the dereferenced size
        # value instead of trying to calculate the size each time.
        #
        return .__size
    end

    #MD:
    #: Concatenate(ustr) - this will concatenate the data from this object followed
    #: by the data from the ustr and return a new UTF-8 string object with the
    #: new concatenated data. It uses the UTF8::UTF8.Concatenate() method to do this.
    #:
    #:@param ustr       this value should be something that can be coerced into
    #:@                 a UTF-8 string object
    #:@returns          a new UTF-8 string object whose data is the concatenation
    #:@                 of the data in the current object and the data in the
    #:@                 supplied object
    #:@aborts           if the  supplied object cannot be coerced into a UTF-8
    #:@                 string object
    #:
    method Concatenate(ustr)
        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             UTF8::UTF8.__AuxMessage(103))

        DebugPrint("self:", ::image(self), "ustr:", ::image(ustr))

        return UTF8::UTF8.Concatenate(self, ustr)
    end

    #MD:
    #: Subsection(i, j) - this method will return a new UTF-8 object consisting
    #: of those codepoints that correspond to the codepoints between position i
    #: and position j. The positions are as per normal unicon/icon strings.
    #:
    #:@param i          if it is not supplied, it will default to 1, if it is
    #:@                 negative, the value will be based on the size of the
    #:@                 the UTF-8 string. the same referencing rules as for
    #:@                 unicon/icon strings - RHS start for negative numbers
    #:param j           if it is not supplied, it will default to i + 1, if it is
    #:@                 negative, the value will be based on the size of the
    #:@                 the UTF-8 string. the same referencing rules as for
    #:@                 unicon/icon strings - RHS start for negative numbers
    #:@returns          a new UTF-8 object representing the codepoints found
    #:@                 between i and j positions
    #:@fails            if the values of i and j are outside the UTF-8 string
    #:@                 the method will fail
    #:
    method Subsection(i, j)

        DebugPrint("self:", image(self), " i:", i, " j:", j)

        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          UTF8::UTF8.__AuxMessage(101))

        /j & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          UTF8::UTF8.__AuxMessage(101))

        return UTF8::UTF8.Subsection(self, i, j)
    end

    #MD:
    #:
    #:
    method Subscript(i)
        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          UTF8::UTF8.__AuxMessage(101))

        return UTF8::UTF8.Subscript(self, i)
    end

    #MD:
    #: String() - return the data stored in the __data field
    #:
    method String()
        return .__data
    end


    #MD:
    #: Invalid() - this method will return the original invalid source data only
    #: in the situation where there were found to be invalid UTF-8 codepoints bytes,
    #: otherwise, it will fail.
    #:
    #:@returns          the dereferenced source string if there were any invalid
    #:@                 found.
    #:@fails            if the UTF-8 string consists of only valid bytes in the
    #:@                 correct order in the string
    #:
    method Invalid()
        return .\__source_data
    end

#MD:
#: initially(val, convert) - this method will create a new UTF-8 string. It will handle
#: the coercion of other unicon values to the UTF-8 string, including coercing
#: unicon strings that have byte values in the range of 128 to 255. Any strings
#: that have such byte values will be converted to the multi-byte characters only if
#: the convert parameter is non-null. Otherwise, any byte values in the range of
#; 128 to 255 will be assumed to be part of a UTF-8 codepoint.
#:
#: Since the object will be treated as immutable, the size of the resulting string
#: will be calculated, any invalid codepoints will be translated to the UTF-8 Invalid
#: Codepoint value. However, the original raw byte string will be kept in the
#: background for analysis, if required. The additional field will be normally
#: null for all valid UTF-8 strings.
#:
#:@param val            the value to be used as data for the UTF-8 string
#:@param convert        used only when unicon string are to be converted from
#:@                     a sequence of bytes to the corresponding UTF-8 codepoints
#:@returns              a new UTF-8 codepoint string
#:
initially(val, convert)
    local   byte,                       #LV: temporary to hold the next byte
                                        #:   from the source as we do the initial
                                        #:   analysis of the bytes
            tempstr                     #LV: temporary to hold a translated byte
                                        #:   string to a UTF-8 encoded string

    #
    # We will test if the global debugging is turned on or off.
    #
    IsOn()
    #
    # we will start our analysis with determining if the val is a unicon/icon
    # string or cset and if it is to be treated as a UTF-8 byte stream or a non-UTF-8
    # byte stream.
    #
    if ::type(val) == ("string" | "cset") then {

        DebugPrint("(", &level, ") type of val: ", ::type(val), " value of val: ", ::ximage(val))

        #
        # if we are to convert the val from a byte string to a UTF-8 string, we
        # do so by translating each byte to the corresponding codepoint.
        #
        if \convert | ::type(val) == "cset" then {
            tempstr := ""
            val := ::string(val)
            every byte := !val do {
                tempstr ||:= UTF8::UTF8.ByteTranslate(byte)
            }
            val := tempstr
        }
        #
        # calculate the number of codepoints in val by checking each byte against
        # __UTF8CountTable.
        #
        __size := UTF8::UTF8.Size(val)

        DebugPrint("(", &level, ") Size of val: ", __size)

        #
        # if the val is made up of Valid UTF-8 codepoints then just assign val to
        # the __data field and calculate the size of the UTF-8 string
        #

        DebugPrint("Is Valid: ", image(UTF8::UTF8.Valid(val)) | "fail")

        if __data := (UTF8::UTF8.Valid(val)) then {
            #
            # It is ASCII only if the number of bytes is the same as the number of
            # codepoints. We are able to set the size of the UTF-8 string at the
            # same time we test for it being ASCII
            #

            DebugPrint("(", &level, ") pretest - Is ASCII only: ", \__asciionly | "null", " __data:", ::image(__data))

            __asciionly := ((*val = __size) & 1 )

            DebugPrint("(", &level, ") posttest - Is ASCII only: ", \__asciionly | "null", " __data:", ::image(__data))
            DebugPrint("(", &level, ") __data:", ::image(__data))

        #
        # otherwise we have some invalid codepoint and we want these to be
        # transformed into the UTF-8 Invalid Codepoint codepoint. We will also
        # keep a copy of the original string for later analysis if required
        #
        } else {
            __source_data := val

            DebugPrint("__source_data:", ::image(__source_data))

            __data := ""
            every __data ||:= UTF8::UTF8.ForEach(val).String()

            DebugPrint("__data:", ::image(__data))
        }

    #
    # otherwise we check to see if it is a standard unicon/icon number value
    # which we will convert directly to a string. There are not expected to be
    # any problems here as each digit/element of a unicon/icon number converts
    # to its corresponding single byte representation in the UTF-8 codepoints
    #
    } else if ::type(val) == ("integer" | "real") then {
        __data := ::string(val)
        __size := *__data
        __asciionly := 1

    #
    # otherwise, we check to see if it is already a UTF-8 object and simply
    # return that value as we are treating all UTF-8 strings as immutable.
    #
    ################################ IMPLEMENTATION NOTE########################
    #
    # Due to the implementation of objects and classes within Unicon, we need to assign
    # to the variable "self" in the procedure that has called this in the underlying
    # implementation. To do this we need to access that specific variable on the
    # "call stack" and we use the function "variable()" to achieve this. Note that
    # the call stack is not as it appears within this code.
    #
    } else if IsObjectOf(val, "UTF8") then {
        variable("self", , 1) := val

    #
    # otherwise, we check to see if it is a UTF-8 cset value and return the
    # corresponding UTF-8 string in like manner to the normal unicon/icon cset
    # to string conversion process. As UTF-8 csets are also immutable, they contain
    # a representation of themselves as a UTF-8 string and we simply return this
    # value, instead of creating a new UTF-8 object.
    #
    ################################ IMPLEMENTATION NOTE########################
    #
    # Due to the implementation of objects and classes within Unicon, we need to assign
    # to the variable "self" in the procedure that has called this in the underlying
    # implementation. To do this we need to access that specific variable on the
    # "call stack" and we use the funation "variable" to achieve this. Note that
    # the call stack is not as it appears within this code.
    #
    } else if IsObjectOf(val, "UTF8Set") then {
        variable("self", , 1) := val.UTF8Value()

    #
    # finally, we have some value that cannot be coerce into a UTF-8 string, so
    # we do what the normal unicon/icon runtime does and runtime abort with a
    # 103 - not a string error
    #
    } else {
        AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                     "\noffending value:", ::image(val))
    }

    DebugPrint("This object is ", ::image(self))
end

# class UTF8Set
class UTF8Set : ClassClass(
    __ascii_blank,                      #CV: this is the UTF-8 cset that consists
                                        #:   of a single ASCII " "

    __uft8_blanks,                      #CV: this is the UTF-8 cset that consists
                                        #:   of all the current (as at 23-Sept-2018)
                                        #:   UTF-8 blank codepoints as follows from
                                        #:
                                        #:   https://en.wikipedia.org/wiki/Whitespace_character
                                        #:
                                        #:   U+0020     space               " "
                                        #:   U+00A0     no-break space      " "
                                        #:   U+1680     ogham space mark    " "
                                        #:   U+2000     en quad             " "
                                        #:   U+2001     em quad             " "
                                        #:   U+2002     en space            " "
                                        #:   U+2003     em space            " "
                                        #:   U+2004     3-per-em space      " "
                                        #:   U+2005     4-per-em space      " "
                                        #:   U+2006     6-per-em space      " "
                                        #:   U+2007     figure space        " "
                                        #:   U+2008     punctuation space   " "
                                        #:   U+2009     thin space          " "
                                        #:   U+200A     hair space          " "
                                        #:   U+202F     narrow no-break space
                                        #:                                  " "
                                        #:   U+205F     medium mathematical space
                                        #:                                  " "
                                        #:   U+3000     ideographic space   "　"

    __all_codepoints,                   #CV: this is the UTF-8 cset that represents
                                        #:   all UTF-8 codepoints

    __ascii_lparens,                    #CV: this is the UTF-8 cset that represents
                                        #:   the single ASCII "("

    __ascii_rparens,                    #CV: this is the UTF-8 cset that represents
                                        #:   the single ASCII ")"

    __ascii_128,                        #CV: this is the UTF-8 cset that represents
                                        #:   the 128 ASCII codepoints
    __defined_sets,                     #CV: table keyed by string to value of
                                        #:   UFT8Set Object
    __max_size_of_set                   #CV: this is the maximum number of codepoints
                                        #:   that currently exist in the UTF-8
                                        #:   space.
)

    method __Coerce(val)
        /val & AbortMessage(2001, keyword("&file", , 1), keyword("&line", , 1),
        "\noffending value: &null\nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value.")

        if IsObjectOf(val, "UTF8Set") then {
            return val
        } else {
            return UTF8::UTF8Set.New(val)
        }
    end

    #MD:
    #: __AuxMessage(errorno, messageno)
    #:
    #:
    method __AuxMessage(errorno, messageno)
        static  ErrorMessages           #SV: table holding error messages for error
                                        #:   numbers used in UFT8/UTF8Set aborts

        initial {
            ErrorMessages := ::table()

            ErrorMessages[103 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8 object/value or object/value that can be converted to an UTF8 object/value."
            ErrorMessages[103 || "-" || 2] := "\noffending value: "
            ErrorMessages[103 || "-" || 3] := "\noffending value: &null_
                                               \nMissing Unicon string value"
            ErrorMessages[103 || "-" || 4] := "\noffending value: &null_
                                               \nMissing Unicon integer value or Unicon string value"
            ErrorMessages[104 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[104 || "-" || 2] := "\noffending value: &null"
            ErrorMessages[110 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8 object/value or object/value that can be converted to an UTF8 object/value."
            ErrorMessages[112 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[113 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8 object/value or object/value that can be converted to an UTF8 object/value."
            ErrorMessages[114 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[116 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[120 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[122 || "-" || 1] := "\noffending value: &null_
                                               \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
            ErrorMessages[122 || "-" || 2] := "\noffending value: "
            ErrorMessages[2001 || "-" || 1] := "\noffending value: &null_
                                                \nMissing UTF8Set object/value or object/value that can be converted to an UTF8Set object/value."
        }
        /messageno := 1
        return \ErrorMessages[errorno || "-" || messageno] | \(self$ClassClass.__AuxMessage(errorno, messageno))
    end

    method Complement(uset)
        local   local_uset              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /uset & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(120))

        local_uset := __Coerce(uset)
        return local_uset.Complement()
    end

    method Difference(uset1, uset2)
        local   local_uset1             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /uset1 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(120))

        /uset2 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(120))

        local_uset1 := __Coerce(uset1)
        return local_uset1.Difference(uset2)
    end

    method Intersect(uset1, uset2)
        local   local_uset1             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.
        /uset1 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(120))

        /uset2 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(120))

        local_uset1 := __Coerce(uset1)
        return local_uset1.Intersect(uset2)
    end

    method Union(uset1, uset2)
        local   local_uset1             #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /uset1 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(120))

        /uset2 & AbortMessage(120, keyword("&file", , 1), keyword("&line", , 1),
                              __AuxMessage(120))

        DebugPrint("image(uset1):", image(uset1), "\nimage(uset2):", image(uset2))

        local_uset1 := __Coerce(uset1)

        DebugPrint("image(local_uset1):", image(local_uset1))

        return local_uset1.Union(uset2)
    end

    method ForEach(val)
        local   local_val               #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /val & AbortMessage(116, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(116))

        local_val := __Coerce(val)
        suspend local_val.ForEach()
    end

    method Size(val)
        local   local_val               #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /val & AbortMessage(112, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(112))

        local_val := __Coerce(val)
        return local_val.Size()
    end

    #MD:
    #: String(val) - return the string representation of the val give after
    #: coercing to a UTF8Set value
    #:
    method String(val)
        /val & return ""
        return __Coerce(val).String()
    end

    #MD:
    #:
    #:
    method Subscript(val, i)
        local   local_val               #LV:

        /val & AbortMessage(114, keyword("&file", , 1), keyword("&line", , 1),
                            __AuxMessage(114))

        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          __AuxMessage(101))

        local_val := __Coerce(val)
        return local_val.Subscript(i)
    end

    method Member(uset, val[])
        local   local_uset              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

        /uset & AbortMessage(122, keyword("&file", , 1), keyword("&line", , 1),
                             __AuxMessage(122))

        not (::type(uset) == "cset" | IsObjectOf(uset, "UTF8Set")) &
            AbortMessage(122, keyword("&file", , 1), keyword("&line", , 1),
                         __AuxMessage(122, 2), image(uset))

        local_uset := __Coerce(uset)

        DebugPrint("Size of val:", *val, "\n", "image(val):", ::image(val), "\n",
                   "coerced parameter uset:", ::image(uset))

        (*val = 0) & return uset
        return (local_uset.Member ! val) & uset
    end

    #MD:
    #: Space() - this returns the UTF-8 set that represents the ASCII space
    #: character
    #:
    #:@returns          UTF8Set object representing the ASCII space character
    #:
    method Space()
        return .__ascii_blank
    end

    #MD:
    #: UTF8Spaces() this returns the UTF-8 set that represents all of the UTF-8
    #: space codepoints. See class header for details.
    #:
    #:@returns          UTF8Set object representing all of the UTF-8 space codepoints
    #:
    method UTF8Spaces()
        return .__uft8_blanks
    end

    #MD:
    #: Ascii() - return the UTF-8 cset that contains only the 128 ASCII codepoints
    #:
    #:@returns          UTF8Set object representing the 128 ASCII codepoints
    method Ascii()
        return .__ascii_128
    end

    #MD:
    #: Rparens() - return the UTF-8 cset that contains only the ')' codepoint
    #:
    #:@returns          UTF8Set object representing the ')' codepoint
    method Rparens()
        return .__ascii_rparens
    end

    #MD:
    #: Lparens() - return the UTF-8 cset that contains only the '(' codepoint
    #:
    #:@returns          UTF8Set object representing the '(' codepoint
    method Lparens()
        return .__ascii_lparens
    end

    #MD:
    #: All() - return the UTF-8 cset that contains all codepoints
    #:
    #:@returns          UTF8Set object representing all codepoints
    method All()
        return .__all_codepoints
    end

    #MD:
    #: MaxSize() - returns the maximum number of codepoints in the UTF-8 space
    #:
    #:@returns          a number
    method MaxSize()
        return .__max_size_of_set
    end

    #MD:
    #: Defined(str) - return the previously defined UFT8Set object based on the
    #: supplied string/cset value. If the value is either a UTF8 string object or
    #: UFT8Set object, the appropriate object will be returned.
    #:
    method Defined(str)
        local   temp                    #LV:

        /str & fail
        if IsObjectOf(str, "UTF8Set") then {
            return str
        } else if IsObjectOf(str ,"UTF8") then {
            if temp := \__defined_sets[str.String()] then {
                return temp
            } else {
                return __defined_sets[str.String()] := str
            }
        } else if ::type(str) == ("string" | "cset" | "integer" | "real") then {
                if temp := \__defined_sets[::string(::cset(str))] then {
                    return temp
                } else {
                    return __defined_sets[::string(::cset(str))] := UTF8Set.New(::string(::cset(str)))
                }
        }
    end

initially()
    initial {
        Execute(ClassObject::ClassClass)
        #
        # We will test if the global debugging is turned on or off.
        #
        IsOn()
        SetClass(UTF8::__UTF8SetObject)
        UTF8::UTF8Set := self
        __defined_sets := table()
        __defined_sets[::string(&ascii)] := __ascii_blank := New(::string(&ascii))

        DebugPrint("__ascii_128 is ", ::image(__ascii_blank))

        __defined_sets[" "] := __ascii_blank := New(" ")

        DebugPrint("__ascii_blank is ", ::image(__ascii_blank))

        __defined_sets["                　"] := __uft8_blanks := New("                　")   # there are 17 UTF-8 blank codepoints

        DebugPrint("__uft8_blanks is ", ::image(__uft8_blanks))

        __defined_sets["("] := __ascii_lparens := New("(")

        DebugPrint("__ascii_lparens is ", ::image(__ascii_lparens))

        __defined_sets[")"] := __ascii_rparens := New(")")

        DebugPrint("__ascii_rparens is ", ::image(__ascii_rparens))

        __all_codepoints := New()

        DebugPrint("__all_codepoints is ", ::image(__all_codepoints))

        __max_size_of_set := 1112064
        ::runtime_error_message(2001, "cannot coerce to UTF8Set object/value")

        DebugPrint("Have Finished Initialising the Class UTF8Set")
    }
end

# class __UTF8SetObject
class __UTF8SetObject : ClassObject(
    __allcp,                            #CV: when not &null, the set of UTF8
                                        #:   codepoints will be dynamically
                                        #:   calculated
    __cset,                             #CV: this is the set of single byte
                                        #:   codepoints applicable. This is a "set"
                                        #:   value.
    __2bytedata,                        #CV: this is the set of two byte codepoints
                                        #:   applicable. This is a "set"
                                        #:   value.
    __3bytedata,                        #CV: this is the set of three byte codepoints
                                        #:   applicable. This is a "set"
                                        #:   value.
    __4bytedata,                        #CV: this is the set of four byte codepoints
                                        #:   applicable. This is a "set"
                                        #:   value.
    __negated,                          #CV: when set, the various set values
                                        #:   represent a negated value
    __string_representation             #CV: this is a valid UTF-8 string that
                                        #:   represents the UTF-8 cset in the
                                        #:   same manner as normal unicon/icon
                                        #:   strings. we can do this as we treat
                                        #:   these objects as immutable.
)
    #: As there is no simple way to represent all codepoints from the UTF-8
    #: standard, I have made the object class represent this UTF-8 cset by the __allcp
    #: flag being set to a non-null value. In doing so, each of the sets to represent the
    #: different sized codepoints is set to &null.
    #:
    #: When the object is created, the value supplied is coerced into an appropriate
    #: form. This is then used to populate the various sets. These sets represent the
    #: various sized codepoints.

    #MD:
    #: __DebugPrint(modifier, val, uset) - this is an internal helper function
    #: for debugging purposes only. modifier is a string that indicates what
    #: procedure the call was made in. val and uset are UTF-8 csets that are
    #: in used in the calling procedure. These will always exist in terms of the
    #: procedure which calls this helper function.
    #:
    #:@param modifier   string value to make it easier to remember where the
    #:@                 specific debugging messages have come from
    #:@param val        this is the UTF-8 cset object that is being operated
    #:@                 on in the calling procedure. In use, it will always exist
    #:@param uset       this is the temporary UTF-8 cset object that has been
    #:@                 created in the calling procedure. In use, it will always
    #:@                 exist.
    #:@fails            procedure will always fail
    #:
    method __DebugPrint(modifier, val, uset)
        #
        # if modifier has been given a value, turn into a string and append a
        # space to the value. This space will be used to separate the modifier
        # from the rest of the debugging message.
        #
        \modifier := modifier || " "

        #
        # if no modifier is supplied, change the value to the empty string for
        # display. We do the null test after the non-null test to ensure that
        # we don't add any extraneous space. If the order of tests was reversed,
        # we would always append a space and this would affect the message on
        # output.
        #
        /modifier := ""

        DebugPrint(modifier, "self.__allcp:", ::ximage(__allcp), "\n",
                   modifier, "self.__cset:", ::ximage(__cset), "\n",
                   modifier, "self.__2bytedata:", ::ximage(__2bytedata), "\n",
                   modifier, "self.__3bytedata:", ::ximage(__3bytedata), "\n",
                   modifier, "self.__4bytedata:", ::ximage(__4bytedata), "\n",
                   modifier, "self.__negated:", ::ximage(__negated), "\n",
                   modifier, "self.__string_representation.String():", self.__string_representation.String(), "\n",
                   modifier, "val.__allcp:", ::ximage(val.__allcp), "\n",
                   modifier, "val.__cset:", ::ximage(val.__cset), "\n",
                   modifier, "val.__2bytedata:", ::ximage(val.__2bytedata), "\n",
                   modifier, "val.__3bytedata:", ::ximage(val.__3bytedata), "\n",
                   modifier, "val.__4bytedata:", ::ximage(val.__4bytedata), "\n",
                   modifier, "val.__negated:", ::ximage(val.__negated), "\n",
                   modifier, "val.__string_representation.String():", val.__string_representation.String(), "\n",
                   modifier, "uset.__allcp:", ::ximage(uset.__allcp), "\n",
                   modifier, "uset.__cset:", ::ximage(uset.__cset), "\n",
                   modifier, "uset.__2bytedata:", ::ximage(uset.__2bytedata), "\n",
                   modifier, "uset.__3bytedata:", ::ximage(uset.__3bytedata), "\n",
                   modifier, "uset.__4bytedata:", ::ximage(uset.__4bytedata), "\n",
                   modifier, "uset.__negated:", ::ximage(uset.__negated), "\n",
                   modifier, "uset.__string_representation.String():", uset.__string_representation.String())
    end

    #MD:
    #: Equiv(val) - tests if the supplied value is the same value as this object
    #:
    method Equiv(uset)
        /uset & fail
        return if ::image(self) == ::image(uset) then {
            return uset
        } else if IsObjectOf(uset, "Utf8Set") then {
            self.__allcp === uset.__allcp | fail
            self.__negated === uset.__negated | fail
            equiv(self.__cset, uset.__cset) | fail
            equiv(self.__2bytedata, uset.__2bytedata) | fail
            equiv(self.__3bytedata, uset.__3bytedata) | fail
            equiv(self.__4bytedata, uset.__4bytedata) | fail
            return uset
        } else {
            fail
        }
    end

    method Nequiv(uset)
        /uset & fail
        return not Equiv(uset) & uset
    end

    #MD:
    #:
    #:
    method LexEquals(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        return UTF8::UTF8.LexEquals(self.UTF8Value(), str)
    end

    #MD:
    #:
    #:
    method LexGTorEq(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        return UTF8::UTF8.LexGTorEq(self.UTF8Value(), str)
    end

    #MD:
    #:
    #:
    method LexGT(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        return UTF8::UTF8.LexGT(self.UTF8Value(), str)
    end

    #MD:
    #:
    #:
    method LexLTorEq(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        return UTF8::UTF8.LexLTorEq(self.UTF8Value(), str)
    end

    #MD:
    #:
    #:
    method LexLT(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        return UTF8::UTF8.LexLT(self.UTF8Value(), str)
    end

    #MD:
    #:
    #:
    method LexNE(str)
        /str & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        return UTF8::UTF8.LexNE(self.UTF8Value(), str)
    end

    method Complement()
        local   cnt,                    #LV: temporary to contain total count
                                        #:   of codepoints within this UTFSet
                temp,
                data

        DebugPrint("self:", ::image(self), "\n__allcp:", ::image(__allcp), "\n__negated:", ::image(__negated),
                   "\n__cset:", ::ximage(__cset), "\n__2bytedata:", ::ximage(__2bytedata),
                   "\n__3bytedata:", ::ximage(__3bytedata), "\n__4bytedata:", ::ximage(__4bytedata),
                   "\n__string_representation:", ::image(__string_representation.String()))

        #
        # we need to count to determine how we will set the flags on negation. As
        # we can represent the entire set by one flag __allcp and the total count
        # must be 0. In this case, negation is indicated by changing the __allcp
        # value to/from 1/&null. the __negated flag will remain &null in this case.
        # However, if the count is above zero, negation is a more complicated process
        #
        temp := UTF8::UTF8Set.New("")
        cnt := *__cset + *__2bytedata + *__3bytedata + *__4bytedata
        if cnt = 0 then {
            temp.__negated := &null
            temp.__allcp := (\self.__allcp & &null) | 1
            temp.__string_representation := UTF8Set.SpecialString((\self.__allcp & "-- Full UTF-8 codepoint set (1,112,064 codepoints) --") | "")

        } else {
            temp.__allcp := &null
            temp.__negated := (\self.__negated & &null) | 1
            temp.__cset := self.__cset
            temp.__2bytedata := self.__2bytedata
            temp.__3bytedata := self.__3bytedata
            temp.__4bytedata := self.__4bytedata
            data := ""
            every data ||:= !::sort(temp.__cset ++ temp.__2bytedata ++ temp.__3bytedata ++ temp.__4bytedata)
            temp.__string_representation := UTF8::UTF8.New((\temp.__negated & "-- All UTF-8 codepoints except (" || data || ") --") | data)
        }

        DebugPrint("temp:", ::image(temp), "\ntemp.__allcp:", ::image(temp.__allcp), "\ntemp.__negated:", ::image(temp.__negated),
                   "\ntemp.__cset:", ::ximage(temp.__cset), "\n__2bytedata:", ::ximage(temp.__2bytedata),
                   "\ntemp.__3bytedata:", ::ximage(temp.__3bytedata), "\ntemp.__4bytedata:", ::ximage(temp.__4bytedata),
                   "\ntemp.__string_representation:", ::image(temp.__string_representation.String()))

        return temp
    end

    method Difference(val)
        local   local_val,              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                uset,                   #LV: temporary to hold result
                temp                    #LV: temporary to hold unicon string

        /val & AbortMessage(114, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(114))

        local_val := UTF8::UTF8Set.__Coerce(val)
        uset := UTF8::UTF8Set.New("")

        __DebugPrint(, local_val, uset)

        #
        # We need to calulate the flags and values of the resultant UTFSet. There
        # are only a small number of valid combinations of flags that will occur
        # these are (S - self, L - val supplied, U - resultant value calculated)
        #           (fields: AC - __allcp, N - __negated, C - selfsum/valsum)
        #           (applicable values: not set - &null, set - 1, don't care - X)
        #
        #   S.AC  |  L.AC  |  S.N   |  L.N   |  S.C   |  L.C   ||  U.AC  |  U.N   |  U.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  | &null  | &null  |   X    |   X    |]  &null | &null  | S.sets -- L.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null |   1    |   X    |   X    |   X    |   X    |]  &null | &null  |
        #     1   | &null  |   X    |   X    |   X    |   X    |]  &null |   1    | L.sets
        #     1   |   1    |   X    |   X    |   X    |   X    |]  &null | &null  |
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  | &null  |   1    |  >=0   |  >=0   |]  &null |   1    | S.sets ** L.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  |   1    | &null  |  >=0   |  >=0   |]  &null |   1    | L.sets ++ S.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  |   1    |   1    |  >=0   |  >=0   |]  &null |   1    | L.sets -- S.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #
        if (\self.__allcp | \local_val.__allcp) then {

            DebugPrint("(A)self.__allcp:", image(self.__allcp), " local_val.__allcp:", image(local_val.__allcp))

            if \self.__allcp & /local_val.__allcp then {
                uset.__allcp := &null
                uset.__negated := 1
                uset.__cset := local_val.__cset
                uset.__2bytedata := local_val.__2bytedata
                uset.__3bytedata := local_val.__3bytedata
                uset.__4bytedata := local_val.__4bytedata
                temp := ""
                every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                uset.__string_representation := UTF8::UTF8.New("-- All UTF-8 codepoints except (" || temp || ") --")

            } else {
                uset.__allcp := &null
                uset.__negated := &null
                temp := ""
                uset.__string_representation := UTF8.SpecialString(temp)

            }
        } else if /self.__allcp & /self.__negated & /local_val.__allcp & /local_val.__negated then {

            DebugPrint("(B)self.__allcp:", image(self.__allcp), " self.__negated:", image(self.__negated),
                       "\nlocal_val.__allcp:", image(local_val.__allcp), " local_val.__negated:", image(local_val.__negated))

            uset.__allcp := &null
            uset.__negated := &null
            uset.__cset := self.__cset -- local_val.__cset
            uset.__2bytedata := self.__2bytedata -- local_val.__2bytedata
            uset.__3bytedata := self.__3bytedata -- local_val.__3bytedata
            uset.__4bytedata := self.__4bytedata -- local_val.__4bytedata
            temp := ""
            every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
            uset.__string_representation := UTF8::UTF8.New(temp)

        } else if /self.__allcp & /local_val.__allcp & (\self.__negated | \local_val.__negated) then {

            DebugPrint("(C)self.__allcp:", image(self.__allcp), " self.__negated:", image(self.__negated),
                       "\nlocal_val.__allcp:", image(local_val.__allcp), " local_val.__negated:", image(local_val.__negated))

            if /self.__negated & \local_val.__negated then {
                uset.__allcp := &null
                uset.__negated := &null
                uset.__cset := self.__cset ** local_val.__cset
                uset.__2bytedata := self.__2bytedata ** local_val.__2bytedata
                uset.__3bytedata := self.__3bytedata ** local_val.__3bytedata
                uset.__4bytedata := self.__4bytedata ** local_val.__4bytedata
                temp := ""
                every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                uset.__string_representation := UTF8::UTF8.New(temp)

            } else if \self.__negated & /local_val.__negated then {
                    uset.__allcp := &null
                    uset.__negated := 1
                    uset.__cset := self.__cset ++ local_val.__cset
                    uset.__2bytedata := self.__2bytedata ++ local_val.__2bytedata
                    uset.__3bytedata := self.__3bytedata ++ local_val.__3bytedata
                    uset.__4bytedata := self.__4bytedata ++ local_val.__4bytedata
                    temp := ""
                    every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                    uset.__string_representation := UTF8::UTF8.New("-- All UTF-8 codepoints except (" || temp || ") --")

            } else if \self.__negated & \local_val.__negated then {
                uset.__allcp := &null
                uset.__negated := &null
                uset.__cset := local_val.__cset -- self.__cset
                uset.__2bytedata := local_val.__2bytedata -- self.__2bytedata
                uset.__3bytedata := local_val.__3bytedata -- self.__3bytedata
                uset.__4bytedata := local_val.__4bytedata -- self.__4bytedata
                temp := ""
                every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                uset.__string_representation := UTF8::UTF8.New(temp)

            }
        }
        __DebugPrint(, local_val, uset)

        return uset
    end

    method Intersect(val)
        local   local_val,              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                uset,                   #LV: temporary to hold result
                temp                    #LV: temporary to hold unicon string

        /val & AbortMessage(114, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        local_val := UTF8::UTF8Set.__Coerce(val)
        uset := UTF8::UTF8Set.New("")

        #
        # We need to calulate the flags and values of the resultant UTFSet. There
        # are only a small number of valid combinations of flags that will occur
        # these are (S - self, L - val supplied, U - resultant value calculated)
        #           (fields: AC - __allcp, N - __negated, C - selfsum/valsum)
        #           (applicable values: not set - &null, set - 1, don't care - X)
        #
        #   S.AC  |  L.AC  |  S.N   |  L.N   |  S.C   |  L.C   ||  U.AC  |  U.N   |  U.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  | &null  | &null  |   X    |   X    |]  &null | &null  | S.sets ** L.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null |   1    |   X    |   X    |   X    |   X    |]  &null | &null  | S.sets
        #     1   | &null  |   X    |   X    |   X    |   X    |]  &null | &null  | L.sets
        #     1   |   1    |   X    |   X    |   X    |   X    |]    1   | &null  |
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  | &null  |   1    |  >=0   |  >=0   |]  &null |   1    | S.sets -- L.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  |   1    | &null  |  >=0   |  >=0   |]    1   | &null  | L.sets -- S.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  |   1    |   1    |  >=0   |  >=0   |]  &null |   1    | S.sets ++ L.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #
        if (\self.__allcp | \local_val.__allcp) then {

            DebugPrint("(A)self.__allcp:", image(self.__allcp), " local_val.__allcp:", image(local_val.__allcp))

            if \self.__allcp & \local_val.__allcp then {
                uset.__allcp := 1
                temp := "-- Full UTF-8 codepoint set (1,112,064 codepoints) --"
                uset.__string_representation := UTF8.SpecialString(temp)

            } else if \self.__allcp then {
                uset.__allcp := local_val.__allcp
                uset.__negated := local_val.__negated
                uset.__cset := local_val.__cset
                uset.__2bytedata := local_val.__2bytedata
                uset.__3bytedata := local_val.__3bytedata
                uset.__4bytedata := local_val.__4bytedata
                uset.__string_representation := local_val.__string_representation

            } else {
                uset.__allcp := self.__allcp
                uset.__negated := self.__negated
                uset.__cset := self.__cset
                uset.__2bytedata := self.__2bytedata
                uset.__3bytedata := self.__3bytedata
                uset.__4bytedata := self.__4bytedata
                uset.__string_representation := self.__string_representation

            }
        } else if /self.__allcp & /self.__negated & /local_val.__allcp & /local_val.__negated then {

            DebugPrint("(B)self.__allcp:", image(self.__allcp), " self.__negated:", image(self.__negated),
                       "\nlocal_val.__allcp:", image(local_val.__allcp), " local_val.__negated:", image(local_val.__negated))

            uset.__allcp := &null
            uset.__negated := &null
            uset.__cset := self.__cset ** local_val.__cset
            uset.__2bytedata := self.__2bytedata ** local_val.__2bytedata
            uset.__3bytedata := self.__3bytedata ** local_val.__3bytedata
            uset.__4bytedata := self.__4bytedata ** local_val.__4bytedata
            temp := ""
            every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
            uset.__string_representation := UTF8::UTF8.New(temp)

        } else if /self.__allcp & /local_val.__allcp & (\self.__negated | \local_val.__negated) then {

            DebugPrint("(C)self.__allcp:", image(self.__allcp), " self.__negated:", image(self.__negated),
                       "\nlocal_val.__allcp:", image(local_val.__allcp), " local_val.__negated:", image(local_val.__negated))

            if /self.__negated & \local_val.__negated then {
                uset.__allcp := &null
                uset.__negated := &null
                uset.__cset := self.__cset -- local_val.__cset
                uset.__2bytedata := self.__2bytedata -- local_val.__2bytedata
                uset.__3bytedata := self.__3bytedata -- local_val.__3bytedata
                uset.__4bytedata := self.__4bytedata -- local_val.__4bytedata
                temp := ""
                every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                uset.__string_representation := UTF8::UTF8.New(temp)

            } else if \self.__negated & /local_val.__negated then {
                    uset.__allcp := &null
                    uset.__negated := &null
                    uset.__cset := self.__cset -- local_val.__cset
                    uset.__2bytedata := self.__2bytedata -- local_val.__2bytedata
                    uset.__3bytedata := self.__3bytedata -- local_val.__3bytedata
                    uset.__4bytedata := self.__4bytedata -- local_val.__4bytedata
                    temp := ""
                    every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                    uset.__string_representation := UTF8::UTF8.New(temp)

            } else if \self.__negated & \local_val.__negated then {
                uset.__allcp := &null
                uset.__negated := 1
                uset.__cset := self.__cset ++ local_val.__cset
                uset.__2bytedata := self.__2bytedata ++ local_val.__2bytedata
                uset.__3bytedata := self.__3bytedata ++ local_val.__3bytedata
                uset.__4bytedata := self.__4bytedata ++ local_val.__4bytedata
                temp := ""
                every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                uset.__string_representation := UTF8::UTF8.New("-- All UTF-8 codepoints except (" || temp || ") --")
            }
        }

        __DebugPrint(, local_val, uset)

        return uset
    end

    method Union(val)
        local   local_val,              #LV: temporary to hold the coerced value.
                                        #:   We use a local variable to ensure
                                        #:   that we do not affect the actual
                                        #:   parameter supplied via any pointer
                                        #:   semantics that could be involved.
                                        #:   In other words, if the string involves a
                                        #:   variable reference in the calling
                                        #:   environment, we will not affect
                                        #:   that variable value.

                uset,                   #LV: temporary to hold result
                temp                    #LV: temporary to hold unicon string

        /val & AbortMessage(114, keyword("&file", , 1), keyword("&line", , 1),
                            UTF8::UTF8Set.__AuxMessage(103))

        local_val := UTF8::UTF8Set.__Coerce(val)
        uset := UTF8::UTF8Set.New("")

        __DebugPrint("Union: ", local_val, uset)

        #
        # We need to calulate the flags and values of the resultant UTFSet. There
        # are only a small number of valid combinations of flags that will occur
        # these are (S - self, L - val supplied, U - resultant value calculated)
        #           (fields: AC - __allcp, N - __negated, C - selfsum/valsum)
        #           (applicable values: not set - &null, set - 1, don't care - X)
        #
        #   S.AC  |  L.AC  |  S.N   |  L.N   |  S.C   |  L.C   ||  U.AC  |  U.N   |  U.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  | &null  | &null  |   X    |   X    |]  &null | &null  | S.sets ++ L.sets
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null |   1    |   X    |   X    |   X    |   X    |]        |        |
        #     1   | &null  |   X    |   X    |   X    |   X    |]    1   | &null  | U.sets set to empty
        #     1   |   1    |   X    |   X    |   X    |   X    |]        |        |
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  | &null  |   1    |  >=0   |  >=0   |]    1   | &null  | if L.sets -- S.sets is empty
        #         |        |        |        |        |        |]  &null |   1    | L.sets -- S.sets if L.sets -- S.sets is not empty
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  |   1    | &null  |  >=0   |  >=0   |]    1   | &null  | if S.sets -- L.sets is empty
        #         |        |        |        |        |        |]  &null |   1    | S.sets -- L.sets if S.sets -- L.sets is not empty
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #   &null | &null  |   1    |   1    |  >=0   |  >=0   |]    1   | &null  | if S.sets ** L.sets is empty
        #         |        |        |        |        |        |]  &null |   1    | S.sets ** L.sets if S.sets ** L.sets is not empty
        #   ------+--------+--------+--------+--------+--------++--------+--------+-----------
        #
        #
        if (\self.__allcp | \local_val.__allcp) then {
            DebugPrint("(A)self.__allcp:", image(self.__allcp), " local_val.__allcp:", image(local_val.__allcp))

            uset.__allcp := 1

        } else if /self.__allcp & /self.__negated & /local_val.__allcp & /local_val.__negated then {
            DebugPrint("(B)self.__allcp:", image(self.__allcp), " self.__negated:", image(self.__negated),
                       "\nlocal_val.__allcp:", image(local_val.__allcp), " local_val.__negated:", image(local_val.__negated))

            uset.__allcp := &null
            uset.__negated := &null
            uset.__cset := self.__cset ++ local_val.__cset
            uset.__2bytedata := self.__2bytedata ++ local_val.__2bytedata
            uset.__3bytedata := self.__3bytedata ++ local_val.__3bytedata
            uset.__4bytedata := self.__4bytedata ++ local_val.__4bytedata
            temp := ""
            every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
            uset.__string_representation := UTF8::UTF8.New(temp)

        } else if /self.__allcp & /local_val.__allcp & (\self.__negated | \local_val.__negated) then {

            DebugPrint("(C)self.__allcp:", image(self.__allcp), " self.__negated:", image(self.__negated),
                       "\nlocal_val.__allcp:", image(local_val.__allcp), " local_val.__negated:", image(local_val.__negated))

            if /self.__negated & \local_val.__negated then {
                if *(local_val.__cset -- self.__cset) +
                        *(local_val.__2bytedata -- self.__2bytedata) +
                        *(local_val.__3bytedata -- self.__3bytedata) +
                        *(local_val.__4bytedata -- self.__4bytedata) = 0 then {
                    uset.__allcp := 1
                    uset.__negated := &null
                    temp := "-- Full UTF-8 codepoint set (1,112,064 codepoints) --"
                    uset.__string_representation := UTF8.SpecialString(temp)

                } else {
                    uset.__allcp := &null
                    uset.__negated := 1
                    uset.__cset := local_val.__cset -- self.__cset
                    uset.__2bytedata := local_val.__2bytedata -- self.__2bytedata
                    uset.__3bytedata := local_val.__3bytedata -- self.__3bytedata
                    uset.__4bytedata := local_val.__4bytedata -- self.__4bytedata
                    temp := ""
                    every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                    uset.__string_representation := UTF8::UTF8.New("-- All UTF-8 codepoints except (" || temp || ") --")

                }
            } else if \self.__negated & /local_val.__negated then {
                if *(self.__cset -- local_val.__cset) +
                        *(self.__2bytedata -- local_val.__2bytedata) +
                        *(self.__3bytedata -- local_val.__3bytedata) +
                        *(self.__4bytedata -- local_val.__4bytedata) = 0 then {
                    uset.__allcp := 1
                    uset.__negated := &null
                    temp := "-- Full UTF-8 codepoint set (1,112,064 codepoints) --"
                    uset.__string_representation := UTF8.SpecialString(temp)

                } else {
                    uset.__allcp := &null
                    uset.__negated := 1
                    uset.__cset := self.__cset -- local_val.__cset
                    uset.__2bytedata := self.__2bytedata -- local_val.__2bytedata
                    uset.__3bytedata := self.__3bytedata -- local_val.__3bytedata
                    uset.__4bytedata := self.__4bytedata -- local_val.__4bytedata
                    temp := ""
                    every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                    uset.__string_representation := UTF8::UTF8.New("-- All UTF-8 codepoints except (" || temp || ") --")

                }
            } else if \self.__negated & \local_val.__negated then {
                if *(self.__cset ** local_val.__cset) +
                        *(self.__2bytedata ** local_val.__2bytedata) +
                        *(self.__3bytedata ** local_val.__3bytedata) +
                        *(self.__4bytedata ** local_val.__4bytedata) = 0 then {
                    uset.__allcp := 1
                    uset.__negated := &null
                    temp := "-- Full UTF-8 codepoint set (1,112,064 codepoints) --"
                    uset.__string_representation := UTF8.SpecialString(temp)

                } else {
                    uset.__allcp := &null
                    uset.__negated := 1
                    uset.__cset := self.__cset ** local_val.__cset
                    uset.__2bytedata := self.__2bytedata ** local_val.__2bytedata
                    uset.__3bytedata := self.__3bytedata ** local_val.__3bytedata
                    uset.__4bytedata := self.__4bytedata ** local_val.__4bytedata
                    temp := ""
                    every temp ||:= !::sort(uset.__cset ++ uset.__2bytedata ++ uset.__3bytedata ++ uset.__4bytedata)
                    uset.__string_representation := UTF8::UTF8.New("-- All UTF-8 codepoints except (" || temp || ") --")
                }
            }
        }

        __DebugPrint(, local_val, uset)

        return uset
    end

    #MD:
    #: ForEach() - this methods returns each codepoint as a UTF-8 string from the
    #: cosepoints stored within the UTF-8 cset. it utilises a helper method that
    #: returns each UTF-8 codepoint as a unicon string. It converts each raw value
    #: into a UTF-8 string object.
    #:
    method ForEach()
        local   ch,                     #LV:
                data,                   #LV:
                lst,                    #LV:
                i,
                j,
                k,
                l,
                __1

        DebugPrint("Entering", "__allcp:", ::ximage(__allcp),
                   "__negated:", ::ximage(__negated),
                   "__cset:", ::ximage(__cset),
                   "__2bytedata:", ::ximage(__2bytedata),
                   "__3bytedata:", ::ximage(__3bytedata),
                   "__4bytedata:", ::ximage(__4bytedata))

        if \__allcp then {

            DebugPrint("Producing all UTF-8 codepoints")

            every i := 0 to UTF8Set.MaxSize() - 1 do {
                ch := UTF8.DecimalTo(i)

                DebugPrint("codepoint ch:", ::image(ch), " ximage(ch.String()):",
                           ::ximage(ch.String()), "i:", i)

                suspend ch
            }
        } else if \__negated then {

            DebugPrint("Producing negated UTF-8 codepoints",
                       "Producing all UTF-8 codepoints")

            every i := 0 to UTF8Set.MaxSize() - 1 do {
                ch := UTF8.DecimalTo(i)
                if member(__cset | __2bytedata | __3bytedata | __4bytedata, ch.String()) then {
                    next
                } else {
                    suspend ch
                }
            }
        } else {

            DebugPrint("Producing available UTF-8 codepoints")

            lst := []
            lst := ::sort(\__cset)
            every ch := !\lst do {

                DebugPrint("ch:", ::image(ch))

                suspend UTF8.New(ch)
            }

            lst := []
            lst := ::sort(\__2bytedata)

            DebugPrint("lst:", ::ximage(lst))

            every ch := !\lst do {

                DebugPrint("ch:", ::image(ch), " ximage(ch):", ::ximage(ch))

                suspend UTF8.New(ch)
            }

            lst := []
            lst := ::sort(\__3bytedata)
            every ch := !\lst do {
                DebugPrint("ch:", ::image(ch), " ximage(ch):", ::ximage(ch))

                suspend UTF8.New(ch)
            }

            lst := []
            lst := ::sort(\__4bytedata)
            every ch := !\lst do {

                DebugPrint("ch:", ::image(ch), " ximage(ch):", ::ximage(ch))

                suspend UTF8.New(ch)
            }
        }
    end

    method Size()
        local   maxsz,                  #LV:

                sz                      #LV:

        maxsz := UTF8Set.MaxSize()
        sz := (*\__cset | 0) + (*\__2bytedata | 0) + (*\__3bytedata | 0) + (*\__4bytedata | 0)
        (\__allcp) & sz := maxsz
        (\__negated) & sz := maxsz - sz
        return sz
    end

    #MD:
    #:
    #:
    method Subscript(i)
        local   temp                    #LV:

        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          UTF8::UTF8Set.__AuxMessage(101))

        if /__negated then {
            if \__string_representation then {
                return __string_representation.Subscript(i)
            } else {
                return UTF8::UTF8.DecimalTo(i)
            }
        } else {

        }
    end

    #MD:
    #:
    #:
    method Random()
        return UTF8::UTF8.Random(self.UTF8Value())
    end

    #MD:
    #: TabMatch()
    #:
    method TabMatch()
        return UTF8::UTF8.TabMatch(self.UTF8Value())
    end

    #MD:
    #: Concatenate(ustr) - this will concatenate the data from this object followed
    #: by the data from the ustr and return a new UTF-8 string object with the
    #: new concatenated data. It uses the UTF8::UTF8.Concatenate() method to do this.
    #:
    #:@param ustr       this value should be something that can be coerced into
    #:@                 a UTF-8 string object
    #:@returns          a new UTF-8 string object whose data is the concatenation
    #:@                 of the data in the current object and the data in the
    #:@                 supplied object
    #:@aborts           if the  supplied object cannot be coerced into a UTF-8
    #:@                 string object
    #:
    method Concatenate(ustr)
        /ustr & AbortMessage(103, keyword("&file", , 1), keyword("&line", , 1),
                             UTF8::UTF8Set.__AuxMessage(103))

        DebugPrint("self:", ::image(self), "ustr:", ::image(ustr))

        return UTF8::UTF8.Concatenate(self.UTF8Value(), ustr)
    end

    #MD:
    #: Subsection(i, j) - this method will return a new UTF-8 object consisting
    #: of those codepoints that correspond to the codepoints between position i
    #: and position j. The positions are as per normal unicon/icon strings.
    #:
    #:@param i          if it is not supplied, it will default to 1, if it is
    #:@                 negative, the value will be based on the size of the
    #:@                 the UTF-8 string. the same referencing rules as for
    #:@                 unicon/icon strings - RHS start for negative numbers
    #:param j           if it is not supplied, it will default to i + 1, if it is
    #:@                 negative, the value will be based on the size of the
    #:@                 the UTF-8 string. the same referencing rules as for
    #:@                 unicon/icon strings - RHS start for negative numbers
    #:@returns          a new UTF-8 object representing the codepoints found
    #:@                 between i and j positions
    #:@fails            if the values of i and j are outside the UTF-8 string
    #:@                 the method will fail
    #:
    method Subsection(i, j)

        DebugPrint("self:", image(self), " i:", i, " j:", j)

        /i & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          UTF8::UTF8Set.__AuxMessage(101))

        /j & AbortMessage(101, keyword("&file", , 1), keyword("&line", , 1),
                          UTF8::UTF8Set.__AuxMessage(101))

        return UTF8::UTF8.Subsection(self.UTF8Value(), i, j)
    end

    #MD:
    #: Member - procedure to test if c is a member of the UTF8Set. c is coerced
    #: into a string before matching occurs. If c is a member then the object is
    #: returned. this is based on the behaviour of the function member.
    #:
    method Member(c[])
        local   val,                    #LV:

                found := 1,             #LV:

                cp                      #LV:

        *c = 0 & return self

        DebugPrint("Size of c:", *c, "\n", "image(c):", ::image(c))

        every val := !c do {
            if ::type(val) == ("string" | "cset" | "integer" | "real") | IsObject(val, "UTF8" | "UTF8Set") then {
                val := UTF8::UTF8.__Coerce(val)

                DebugPrint("val.String():", val.String())

                every cp := val.ForEach().String() do {

                    DebugPrint("cp:", cp)

                    if \__allcp then {
                        if /__negated then {

                            DebugPrint("All codepoints in set, so is a member.")

                            break
                        } else {

                            DebugPrint("All codepoints not in set, so is not a member.")

                            found := &null
                            break break
                        }
                    } else {

                        DebugPrint("ximage(__cset):", ::ximage(__cset), "\n",
                                   "ximage(__2bytedata):", ::ximage(__2bytedata), "\n",
                                   "ximage(__3bytedata):", ::ximage(__3bytedata), "\n",
                                   "ximage(__4bytedata):", ::ximage(__4bytedata))

                        if ::member(\__cset | \__2bytedata | \__3bytedata | \__4bytedata, cp) then {
                            if /__negated then {
                                # do nothing

                                DebugPrint("Member of one of the sets")

                            } else {

                                DebugPrint("Not Member of one of the sets")

                                found := &null
                                break break
                            }
                        } else {
                            if /__negated then {
                                found := &null

                                DebugPrint("Not Found in any set")

                                break break
                            } else {

                                DebugPrint("Found in negated set")
                            }
                        }
                    }
                }
            } else {
                found := &null

                DebugPrint("Not a value of any required form")

                break
            }
        }

        DebugPrint("found:", ::ximage(found))

        return (\found & self) | fail
    end

    #MD:
    #: UTF8Value() - this will return the stored UTF-8 string representation value.
    #: In the case of a UTF-8 cset that represents all UTF-8 codepoints, it will
    #: be created here the first time this is called and the created UTF-8 string
    #: will then be be stored in the UTF-8 cset object. This is done as it is
    #: unlikely that the full UTF-8 cset value will be required as a string in
    #: normal operations. If it is used, then it will only be created the first
    #: time this is called on that specific object.
    #:
    #:
    method UTF8Value()
        return .__string_representation
    end

    #MD:
    #: String() - this will return the stored UTF-8 string representation value.
    #: In the case of a UTF-8 cset that represents all UTF-8 codepoints, it will
    #: be created here the first time this is called and the created UTF-8 string
    #: will then be be stored in the UTF-8 cset object. This is done as it is
    #: unlikely that the full UTF-8 cset value will be required as a string in
    #: normal operations. If it is used, then it will only be created the first
    #: time this is called on that specific object.
    #:
    #:
    method String()
        return .__string_representation.String()
    end

#MD:
#: initially(val) - this will create a UTF-8 cset for the supplied value. There
#: are a small number of cases for which the required UTF-8 cset is looked up and
#: returned. For all other values that can be converted into a UTF-8 cset, a new
#: object will be returned each time, even if that value has been previously
#: created. Since these values are being treated as immutable, we have a number
#: of options in relation to the creation of the UTF-8 cset values.
#:
initially(val)
    local   data,                       #LV: temporary for holding a cset value
            ch,                         #LV: temporary for holding a byte value
            __1,                        #LV: declaration for list comprehension
                                        #:   expression use. This avoids any errors
                                        #:   when using the compiler -u flag
            __2,                        #LV: declaration for list comprehension
                                        #:   expression use. This avoids any errors
                                        #:   when using the compiler -u flag
            __3,                        #LV: declaration for list comprehension
                                        #:   expression use. This avoids any errors
                                        #:   when using the compiler -u flag
            __4,                        #LV: declaration for list comprehension
                                        #:   expression use. This avoids any errors
                                        #:   when using the compiler -u flag
            temp                        #LV: temporary for holding a string value

    #
    # We will test if the global debugging is turned on or off.
    #
    IsOn()

    DebugPrint("type of val:", ::type(val),
               "classname of val:", ::classname(val) | "no class name")

    #
    # Since we have made UTF-8 cset objects immutable, we just return any UTF-8
    # cset value that is supplied.
    #
    ################################ IMPLEMENTATION NOTE########################
    #
    # Due to the implementation of objects and classes within Unicon, we need to assign
    # to the variable "self" in the procedure that has called this in the underlying
    # implementation. To do this we need to access that specific variable on the
    # "call stack" and we use the funation "variable" to achieve this. Note that
    # the call stack is not as it appears within this code.
    #
    if IsObjectOf(val, "UTF8Set") then {

        DebugPrint("val: ", val.String())

        variable("self", , 1) := val

    #
    # in the case of a normal unicon/icon cset value being supplied, we need to
    # ensure that the only characters existing in this cset are the 128 ASCII
    # characters. We can test for this by removing all ASCII characters from
    # the cset supplied and that its size is now 0. If there are any characters
    # in the cset >> "\x7f", the size will be > 0, so we won't process it here.
    #
    } else if ::type(val) == "cset" & *(val -- &ascii) = 0 then {

        DebugPrint("type of val:", ::type(val), " and covers the ASCII characters.")

        #
        # the characters that need to be added to the __cset field are those found
        # in the supplied cset value. To obtain these we convert the cset to a string
        # value and then use the list comprehension expression to create a list value
        # to add to the set
        #
        data := ::string(val)
        __cset := ::set([:!data:])
        __2bytedata := ::set([])
        __3bytedata := ::set([])
        __4bytedata := ::set([])
        __string_representation := UTF8::UTF8.SpecialString(::string(val))

    #
    ################################ IMPLEMENTATION NOTE########################
    #
    # Due to the implementation of objects and classes within Unicon, we need to assign
    # to the variable "self" in the procedure that has called this in the underlying
    # implementation. To do this we need to access that specific variable on the
    # "call stack" and we use the funation "variable" to achieve this. Note that
    # the call stack is not as it appears within this code.
    #
        self := UTF8::UTF8Set.Defined()
    #
    # when no value is supplied to the UTFSet creation, this creates a UTFSet value
    # that represents all possible UTF8 codepoints. If the value representing this
    # UTFSet has been created previously, we will use that value.
    #
    } else if /val then {
        #
        # If the value representing this UTFSet has been created previously,
        # we will use that value.
        #

        DebugPrint("type of val:", ::type(val), " value of val: ", ::ximage(val))

        if temp := \UTF8::UTF8Set.All() then {
            #
            ################################ IMPLEMENTATION NOTE########################
            #
            # Due to the implementation of objects and classes within Unicon, we need to assign
            # to the variable "self" in the procedure that has called this in the underlying
            # implementation. To do this we need to access that specific variable on the
            # "call stack" and we use the funation "variable" to achieve this. Note that
            # the call stack is not as it appears within this code.
            #
            variable("self", , 1) := temp

        #
        # Otherwise we will create it
        #
        } else {
            __cset := ::set([])
            __2bytedata := ::set([])
            __3bytedata := ::set([])
            __4bytedata := ::set([])
            __allcp := 1
            __string_representation := UTF8::UTF8.New("-- Full UTF-8 codepoint set (1,112,064 codepoints) --")
        }

    #
    # if the value supplied is an integer or a real then we convert the value
    # first to a Standard Unicon cset to remove any duplicate characters and then
    # we can convert this to a string for creating the proper set using list
    # comprehension. In this case, we will just create a new string representation
    # instead of creating a special string value. If this turns out to be the wrong
    # choice, we can change the call to use the method SpecialString at a later date.
    #
    } else if ::type(val) == ("integer" | "real") then {

        DebugPrint("type of val:", ::type(val), " value of val: ", ::ximage(val))

        data := ::string(::cset(val))
        __cset := ::set([:!data:])
        __2bytedata := ::set([])
        __3bytedata := ::set([])
        __4bytedata := ::set([])
        __string_representation := UTF8::UTF8.New(data)

    #
    # now we will deal with all cset values that have members that are in the upper
    # 128 bytes. We need to extract any of the lower 128 ASCII values and then
    # deal with the other values, which will need conversion from a single byte value
    # to a 2 byte UTF8 copdepoint equivalent. Again we use list comprehension for
    # the extraction of the ascii values.
    #
    } else if ::type(val) == "cset" then {

        DebugPrint("val:", ::ximage(val))

        #
        # extract the relevant ascii values in the cset
        temp := ::string(val ** &ascii)

        DebugPrint("__cset:", ::ximage(temp))

        __cset := ::set([:!temp:])

        #
        # this extracts the upper byte values and uses list comprehension to
        # extract and convert each of the bytes to a proper codepoint
        #
        data := ::string(val -- &ascii)

        DebugPrint("data:", ::ximage(data))

        __2bytedata := ::set([:UTF8::UTF8.ByteTranslate(!data):])
        __3bytedata := ::set([])
        __4bytedata := ::set([])

        #
        # to create the appropriate string representation, we need the sorted union
        # of the __cset and __2bytedata sets as a Unicon string.
        #
        temp := ""
        every temp ||:= !::sort(__cset ++ __2bytedata)

        DebugPrint("__cset ++ __2bytedata:", ::ximage(temp))

        __string_representation := UTF8::UTF8.New(temp)

    #
    # in the case of unicon strings that consist of UTF8 codepoints or if it is
    # a UTF8 object value, we will simply coerce the value to the a UTF8 object
    # value and then extract each codepoint and insert it into the appropriate
    # set based on the size of each codepoint.
    #
    } else if ::type(val) == "string" | IsObjectOf(val, "UTF8") then {

        DebugPrint("level(", &level, ") type of val:", ::type(val), " value of val: ", ::image(val))

        val := UTF8::UTF8.__Coerce(val)

        DebugPrint("level(", &level, ") classname of val:", ::classname(val) | "none", " value of val: ", ::image(val))

        __cset := ::set([])
        __2bytedata := ::set([])
        __3bytedata := ::set([])
        __4bytedata := ::set([])

        DebugPrint("\n__cset:", ::ximage(__cset), "\n__2bytedata:", ::ximage(__2bytedata), "\n__3bytedata:", ::ximage(__3bytedata), "\n__4bytedata:", ::ximage(__4bytedata))

        every data := val.ForEach().String() do {
            case *data of {
                1 : {
                    ::insert(__cset, data)
                }
                2 :{
                    ::insert(__2bytedata, data)
                }
                3 :{
                    ::insert(__3bytedata, data)
                }
                4 :{
                    ::insert(__4bytedata, data)
                }
            }
        }
        temp := ""
        every temp ||:= !::sort(__cset ++ __2bytedata ++ __3bytedata ++ __4bytedata)
        __string_representation := UTF8::UTF8.New(temp)

    #
    # if we reach here then the value supplied cannot be converted to a UTF8Set
    # so we perform a runtime abort
    #
    } else {
        AbortMessage(104, keyword("&file", , 1), keyword("&line", , 1),
                     UTF8::UTF8Set.__AuxMessage(104, 2))
    }
end
