encoder: add edge control characters and fix edge test generation

lib/encoder/encoder.go

@@ -89,12 +89,18 @@ func (mask MultiEncoder) Encode(in string) string {
 		encodeCtl            = uint(mask)&EncodeCtl != 0
 		encodeLeftSpace      = uint(mask)&EncodeLeftSpace != 0
 		encodeLeftTilde      = uint(mask)&EncodeLeftTilde != 0
+		encodeLeftCrLfHtVt   = uint(mask)&EncodeLeftCrLfHtVt != 0
 		encodeRightSpace     = uint(mask)&EncodeRightSpace != 0
 		encodeRightPeriod    = uint(mask)&EncodeRightPeriod != 0
+		encodeRightCrLfHtVt  = uint(mask)&EncodeRightCrLfHtVt != 0
 		encodeInvalidUnicode = uint(mask)&EncodeInvalidUtf8 != 0
 		encodeDot            = uint(mask)&EncodeDot != 0
 	)
 
+	if in == "" {
+		return ""
+	}
+
 	if encodeDot {
 		switch in {
 		case ".":
@@ -110,36 +116,61 @@ func (mask MultiEncoder) Encode(in string) string {
 
 	// handle prefix only replacements
 	prefix := ""
-	if encodeLeftSpace && len(in) > 0 { // Leading SPACE
+	if encodeLeftSpace { // Leading SPACE
 		if in[0] == ' ' {
 			prefix, in = "␠", in[1:] // SYMBOL FOR SPACE
 		} else if r, l := utf8.DecodeRuneInString(in); r == '␠' { // SYMBOL FOR SPACE
 			prefix, in = string(QuoteRune)+"␠", in[l:] // SYMBOL FOR SPACE
 		}
 	}
-	if encodeLeftTilde && len(in) > 0 { // Leading ~
+	if encodeLeftTilde && prefix == "" { // Leading ~
 		if in[0] == '~' {
 			prefix, in = string('~'+fullOffset), in[1:] // FULLWIDTH TILDE
 		} else if r, l := utf8.DecodeRuneInString(in); r == '~'+fullOffset {
 			prefix, in = string(QuoteRune)+string('~'+fullOffset), in[l:] // FULLWIDTH TILDE
 		}
 	}
+	if encodeLeftCrLfHtVt && prefix == "" { // Leading CR LF HT VT
+		switch c := in[0]; c {
+		case '\t', '\n', '\v', '\r':
+			prefix, in = string('␀'+rune(c)), in[1:] // SYMBOL FOR NULL
+		default:
+			switch r, l := utf8.DecodeRuneInString(in); r {
+			case '␀' + '\t', '␀' + '\n', '␀' + '\v', '␀' + '\r':
+				prefix, in = string(QuoteRune)+string(r), in[l:]
+			}
+		}
+	}
 	// handle suffix only replacements
 	suffix := ""
-	if encodeRightSpace && len(in) > 0 { // Trailing SPACE
+	if in != "" {
-		if in[len(in)-1] == ' ' {
+		if encodeRightSpace { // Trailing SPACE
-			suffix, in = "␠", in[:len(in)-1] // SYMBOL FOR SPACE
+			if in[len(in)-1] == ' ' {
-		} else if r, l := utf8.DecodeLastRuneInString(in); r == '␠' {
+				suffix, in = "␠", in[:len(in)-1] // SYMBOL FOR SPACE
-			suffix, in = string(QuoteRune)+"␠", in[:len(in)-l] // SYMBOL FOR SPACE
+			} else if r, l := utf8.DecodeLastRuneInString(in); r == '␠' {
-		}
+				suffix, in = string(QuoteRune)+"␠", in[:len(in)-l] // SYMBOL FOR SPACE
-	}
+			}
-	if encodeRightPeriod && len(in) > 0 { // Trailing .
+		}
-		if in[len(in)-1] == '.' {
+		if encodeRightPeriod && suffix == "" { // Trailing .
-			suffix, in = "．", in[:len(in)-1] // FULLWIDTH FULL STOP
+			if in[len(in)-1] == '.' {
-		} else if r, l := utf8.DecodeLastRuneInString(in); r == '．' {
+				suffix, in = "．", in[:len(in)-1] // FULLWIDTH FULL STOP
-			suffix, in = string(QuoteRune)+"．", in[:len(in)-l] // FULLWIDTH FULL STOP
+			} else if r, l := utf8.DecodeLastRuneInString(in); r == '．' {
+				suffix, in = string(QuoteRune)+"．", in[:len(in)-l] // FULLWIDTH FULL STOP
+			}
+		}
+		if encodeRightCrLfHtVt && suffix == "" { // Trailing .
+			switch c := in[len(in)-1]; c {
+			case '\t', '\n', '\v', '\r':
+				suffix, in = string('␀'+rune(c)), in[:len(in)-1] // FULLWIDTH FULL STOP
+			default:
+				switch r, l := utf8.DecodeLastRuneInString(in); r {
+				case '␀' + '\t', '␀' + '\n', '␀' + '\v', '␀' + '\r':
+					suffix, in = string(QuoteRune)+string(r), in[:len(in)-l]
+				}
+			}
 		}
 	}
+
 	index := 0
 	if prefix == "" && suffix == "" {
 		// find the first rune which (most likely) needs to be replaced
@@ -310,8 +341,10 @@ func (mask MultiEncoder) Decode(in string) string {
 		encodeCtl            = uint(mask)&EncodeCtl != 0
 		encodeLeftSpace      = uint(mask)&EncodeLeftSpace != 0
 		encodeLeftTilde      = uint(mask)&EncodeLeftTilde != 0
+		encodeLeftCrLfHtVt   = uint(mask)&EncodeLeftCrLfHtVt != 0
 		encodeRightSpace     = uint(mask)&EncodeRightSpace != 0
 		encodeRightPeriod    = uint(mask)&EncodeRightPeriod != 0
+		encodeRightCrLfHtVt  = uint(mask)&EncodeRightCrLfHtVt != 0
 		encodeInvalidUnicode = uint(mask)&EncodeInvalidUtf8 != 0
 		encodeDot            = uint(mask)&EncodeDot != 0
 	)
@@ -335,11 +368,15 @@ func (mask MultiEncoder) Decode(in string) string {
 		prefix, in = " ", in[l1:]
 	} else if encodeLeftTilde && r == '～' { // FULLWIDTH TILDE
 		prefix, in = "~", in[l1:]
+	} else if encodeLeftCrLfHtVt && (r == '␀'+'\t' || r == '␀'+'\n' || r == '␀'+'\v' || r == '␀'+'\r') {
+		prefix, in = string(r-'␀'), in[l1:]
 	} else if r == QuoteRune {
 		if r, l2 := utf8.DecodeRuneInString(in[l1:]); encodeLeftSpace && r == '␠' { // SYMBOL FOR SPACE
 			prefix, in = "␠", in[l1+l2:]
 		} else if encodeLeftTilde && r == '～' { // FULLWIDTH TILDE
 			prefix, in = "～", in[l1+l2:]
+		} else if encodeLeftCrLfHtVt && (r == '␀'+'\t' || r == '␀'+'\n' || r == '␀'+'\v' || r == '␀'+'\r') {
+			prefix, in = string(r), in[l1+l2:]
 		}
 	}
 
@@ -347,18 +384,25 @@ func (mask MultiEncoder) Decode(in string) string {
 	suffix := ""
 	if r, l := utf8.DecodeLastRuneInString(in); encodeRightSpace && r == '␠' { // SYMBOL FOR SPACE
 		in = in[:len(in)-l]
-		if r, l2 := utf8.DecodeLastRuneInString(in); r == QuoteRune {
+		if q, l2 := utf8.DecodeLastRuneInString(in); q == QuoteRune {
 			suffix, in = "␠", in[:len(in)-l2]
 		} else {
 			suffix = " "
 		}
 	} else if encodeRightPeriod && r == '．' { // FULLWIDTH FULL STOP
 		in = in[:len(in)-l]
-		if r, l2 := utf8.DecodeLastRuneInString(in); r == QuoteRune {
+		if q, l2 := utf8.DecodeLastRuneInString(in); q == QuoteRune {
 			suffix, in = "．", in[:len(in)-l2]
 		} else {
 			suffix = "."
 		}
+	} else if encodeRightCrLfHtVt && (r == '␀'+'\t' || r == '␀'+'\n' || r == '␀'+'\v' || r == '␀'+'\r') {
+		in = in[:len(in)-l]
+		if q, l2 := utf8.DecodeLastRuneInString(in); q == QuoteRune {
+			suffix, in = string(r), in[:len(in)-l2]
+		} else {
+			suffix = string(r - '␀')
+		}
 	}
 	index := 0
 	if prefix == "" && suffix == "" {

lib/encoder/encoder_test.go

@@ -45,6 +45,22 @@ func TestEncodeSingleMaskEdge(t *testing.T) {
 	}
 }
 
+func TestEncodeDoubleMaskEdge(t *testing.T) {
+	for i, tc := range testCasesDoubleEdge {
+		e := MultiEncoder(tc.mask)
+		t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) {
+			got := e.Encode(tc.in)
+			if got != tc.out {
+				t.Errorf("Encode(%q) want %q got %q", tc.in, tc.out, got)
+			}
+			got2 := e.Decode(got)
+			if got2 != tc.in {
+				t.Errorf("Decode(%q) want %q got %q", got, tc.in, got2)
+			}
+		})
+	}
+}
+
 func TestEncodeInvalidUnicode(t *testing.T) {
 	for i, tc := range []testCase{
 		{

lib/encoder/internal/gen/main.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"log"
 	"math/rand"
@@ -47,21 +48,34 @@ var maskBits = []struct {
 	{encoder.EncodeCtl, "EncodeCtl"},
 	{encoder.EncodeLeftSpace, "EncodeLeftSpace"},
 	{encoder.EncodeLeftTilde, "EncodeLeftTilde"},
+	{encoder.EncodeLeftCrLfHtVt, "EncodeLeftCrLfHtVt"},
 	{encoder.EncodeRightSpace, "EncodeRightSpace"},
 	{encoder.EncodeRightPeriod, "EncodeRightPeriod"},
+	{encoder.EncodeRightCrLfHtVt, "EncodeRightCrLfHtVt"},
 	{encoder.EncodeInvalidUtf8, "EncodeInvalidUtf8"},
 }
-var edges = []struct {
+
+type edge struct {
 	mask    uint
 	name    string
 	edge    int
-	orig    rune
+	orig    []rune
-	replace rune
+	replace []rune
-}{
+}
-	{encoder.EncodeLeftSpace, "EncodeLeftSpace", edgeLeft, ' ', '␠'},
+
-	{encoder.EncodeLeftTilde, "EncodeLeftTilde", edgeLeft, '~', '～'},
+var allEdges = []edge{
-	{encoder.EncodeRightSpace, "EncodeRightSpace", edgeRight, ' ', '␠'},
+	{encoder.EncodeLeftSpace, "EncodeLeftSpace", edgeLeft, []rune{' '}, []rune{'␠'}},
-	{encoder.EncodeRightPeriod, "EncodeRightPeriod", edgeRight, '.', '．'},
+	{encoder.EncodeLeftTilde, "EncodeLeftTilde", edgeLeft, []rune{'~'}, []rune{'～'}},
+	{encoder.EncodeLeftCrLfHtVt, "EncodeLeftCrLfHtVt", edgeLeft,
+		[]rune{'\t', '\n', '\v', '\r'},
+		[]rune{'␀' + '\t', '␀' + '\n', '␀' + '\v', '␀' + '\r'},
+	},
+	{encoder.EncodeRightSpace, "EncodeRightSpace", edgeRight, []rune{' '}, []rune{'␠'}},
+	{encoder.EncodeRightPeriod, "EncodeRightPeriod", edgeRight, []rune{'.'}, []rune{'．'}},
+	{encoder.EncodeRightCrLfHtVt, "EncodeRightCrLfHtVt", edgeRight,
+		[]rune{'\t', '\n', '\v', '\r'},
+		[]rune{'␀' + '\t', '␀' + '\n', '␀' + '\v', '␀' + '\r'},
+	},
 }
 
 var allMappings = []mapping{{
@@ -101,7 +115,7 @@ var allMappings = []mapping{{
 }}
 
 var (
-	rng = rand.New(rand.NewSource(42))
+	rng *rand.Rand
 
 	printables          = runeRange(0x20, 0x7E)
 	fullwidthPrintables = runeRange(0xFF00, 0xFF5E)
@@ -111,6 +125,10 @@ var (
 )
 
 func main() {
+	seed := flag.Int64("s", 42, "random seed")
+	flag.Parse()
+	rng = rand.New(rand.NewSource(*seed))
+
 	fd, err := os.Create("encoder_cases_test.go")
 	fatal(err, "Unable to open encoder_cases_test.go:")
 	defer func() {
@@ -147,29 +165,79 @@ func main() {
 var testCasesSingleEdge = []testCase{
 	`))("Write:")
 	_i = 0
-	for _, e := range edges {
+	for _, e := range allEdges {
-		if _i != 0 {
+		for idx, orig := range e.orig {
-			fatalW(fd.WriteString(" "))("Write:")
+			if _i != 0 {
-		}
+				fatalW(fd.WriteString(" "))("Write:")
-		fatalW(fmt.Fprintf(fd, `{ // %d
+			}
+			fatalW(fmt.Fprintf(fd, `{ // %d
 		mask: %s,
 		in:   %s,
 		out:  %s,
-	},`, i(), e.name, strconv.Quote(string(e.orig)), strconv.Quote(string(e.replace))))("Error writing test case:")
+	},`, i(), e.name, strconv.Quote(string(orig)), strconv.Quote(string(e.replace[idx]))))("Error writing test case:")
+		}
 		for _, m := range maskBits {
-			if len(getMapping(m.mask).src) == 0 {
+			if len(getMapping(m.mask).src) == 0 || invalidMask(e.mask|m.mask) {
 				continue
 			}
-			pairs := buildEdgeTestString(
+			for idx, orig := range e.orig {
-				e.edge, e.orig, e.replace,
+				replace := e.replace[idx]
-				[]mapping{getMapping(0), getMapping(m.mask)}, // quote
+				pairs := buildEdgeTestString(
-				printables, fullwidthPrintables, encodables, encoded, greek) // fill
+					[]edge{e}, []mapping{getMapping(0), getMapping(m.mask)}, // quote
-			for _, p := range pairs {
+					[][]rune{printables, fullwidthPrintables, encodables, encoded, greek}, // fill
-				fatalW(fmt.Fprintf(fd, ` { // %d
+					func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) (out []stringPair) {
+						testL := len(rIn)
+						skipOrig := false
+						for _, m := range testMappings {
+							if runePos(orig, m.src) != -1 || runePos(orig, m.dst) != -1 {
+								skipOrig = true
+								break
+							}
+						}
+						if !skipOrig {
+							rIn[10], rOut[10], quoteOut[10] = orig, orig, false
+						}
+
+						out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
+						for _, i := range []int{0, 1, testL - 2, testL - 1} {
+							for _, j := range []int{1, testL - 2, testL - 1} {
+								if j < i {
+									continue
+								}
+								rIn := append([]rune{}, rIn...)
+								rOut := append([]rune{}, rOut...)
+								quoteOut := append([]bool{}, quoteOut...)
+
+								for _, in := range []rune{orig, replace} {
+									expect, quote := in, false
+									if i == 0 && e.edge == edgeLeft ||
+										i == testL-1 && e.edge == edgeRight {
+										expect, quote = replace, in == replace
+									}
+									rIn[i], rOut[i], quoteOut[i] = in, expect, quote
+
+									if i != j {
+										for _, in := range []rune{orig, replace} {
+											expect, quote = in, false
+											if j == testL-1 && e.edge == edgeRight {
+												expect, quote = replace, in == replace
+											}
+											rIn[j], rOut[j], quoteOut[j] = in, expect, quote
+										}
+									}
+									out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
+								}
+							}
+						}
+						return
+					})
+				for _, p := range pairs {
+					fatalW(fmt.Fprintf(fd, ` { // %d
 		mask: %s | %s,
 		in:   %s,
 		out:  %s,
 	},`, i(), m.name, e.name, strconv.Quote(p.a), strconv.Quote(p.b)))("Error writing test case:")
+				}
 			}
 		}
 	}
@@ -205,9 +273,70 @@ var testCasesSingleEdge = []testCase{
 		mask: EncodeLeftSpace | EncodeRightSpace,
 		in:   "   ",
 		out:  "␠ ␠",
+	}, { // %d
+		mask: EncodeRightPeriod | EncodeRightSpace,
+		in:   "a. ",
+		out:  "a.␠",
+	}, { // %d
+		mask: EncodeRightPeriod | EncodeRightSpace,
+		in:   "a .",
+		out:  "a ．",
 	},
 }
-`, i(), i(), i(), i(), i(), i(), i(), i()))("Error writing test case:")
+
+var testCasesDoubleEdge = []testCase{
+	`, i(), i(), i(), i(), i(), i(), i(), i(), i(), i()))("Error writing test case:")
+	_i = 0
+	for _, e1 := range allEdges {
+		for _, e2 := range allEdges {
+			if e1.mask == e2.mask {
+				continue
+			}
+			for _, m := range maskBits {
+				if len(getMapping(m.mask).src) == 0 || invalidMask(m.mask|e1.mask|e2.mask) {
+					continue
+				}
+				orig, replace := e1.orig[0], e1.replace[0]
+				edges := []edge{e1, e2}
+				pairs := buildEdgeTestString(
+					edges, []mapping{getMapping(0), getMapping(m.mask)}, // quote
+					[][]rune{printables, fullwidthPrintables, encodables, encoded, greek}, // fill
+					func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) (out []stringPair) {
+						testL := len(rIn)
+						for _, i := range []int{0, testL - 1} {
+							for _, secondOrig := range e2.orig {
+								rIn := append([]rune{}, rIn...)
+								rOut := append([]rune{}, rOut...)
+								quoteOut := append([]bool{}, quoteOut...)
+
+								rIn[1], rOut[1], quoteOut[1] = secondOrig, secondOrig, false
+								rIn[testL-2], rOut[testL-2], quoteOut[testL-2] = secondOrig, secondOrig, false
+
+								for _, in := range []rune{orig, replace} {
+									rIn[i], rOut[i], quoteOut[i] = in, in, false
+									fixEdges(rIn, rOut, quoteOut, edges)
+
+									out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
+								}
+							}
+						}
+						return
+					})
+
+				for _, p := range pairs {
+					if _i != 0 {
+						fatalW(fd.WriteString(" "))("Write:")
+					}
+					fatalW(fmt.Fprintf(fd, `{ // %d
+		mask: %s | %s | %s,
+		in:   %s,
+		out:  %s,
+	},`, i(), m.name, e1.name, e2.name, strconv.Quote(p.a), strconv.Quote(p.b)))("Error writing test case:")
+				}
+			}
+		}
+	}
+	fatalW(fmt.Fprint(fd, "\n}\n"))("Error writing test case:")
 }
 
 func fatal(err error, s ...interface{}) {
@@ -224,6 +353,10 @@ func fatalW(_ int, err error) func(...interface{}) {
 	return func(s ...interface{}) {}
 }
 
+func invalidMask(mask uint) bool {
+	return mask&encoder.EncodeCtl != 0 && mask&(encoder.EncodeLeftCrLfHtVt|encoder.EncodeRightCrLfHtVt) != 0
+}
+
 // construct a slice containing the runes between (l)ow (inclusive) and (h)igh (inclusive)
 func runeRange(l, h rune) []rune {
 	if h < l {
@@ -325,11 +458,13 @@ outer:
 	return string(rIn), bOut.String()
 }
 
-func buildEdgeTestString(edge int, orig, replace rune, testMappings []mapping, fill ...[]rune) (out []stringPair) {
+func buildEdgeTestString(edges []edge, testMappings []mapping, fill [][]rune,
+	gen func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) []stringPair,
+) []stringPair {
 	testL := 30
-	rIn := make([]rune, testL)
+	rIn := make([]rune, testL)      // test input string
-	rOut := make([]rune, testL)
+	rOut := make([]rune, testL)     // test output string without quote runes
-	quoteOut := make([]bool, testL)
+	quoteOut := make([]bool, testL) // if true insert quote rune before the output rune
 
 	set := func(i int, in, out rune, quote bool) {
 		rIn[i] = in
@@ -337,6 +472,7 @@ func buildEdgeTestString(edge int, orig, replace rune, testMappings []mapping, f
 		quoteOut[i] = quote
 	}
 
+	// populate test strings with values from the `fill` set
 outer:
 	for pos := 0; pos < testL; pos++ {
 		m := pos % len(fill)
@@ -359,40 +495,26 @@ outer:
 		rOut[i], rOut[j] = rOut[j], rOut[i]
 		quoteOut[i], quoteOut[j] = quoteOut[j], quoteOut[i]
 	})
-	set(10, orig, orig, false)
+	fixEdges(rIn, rOut, quoteOut, edges)
+	return gen(rIn, rOut, quoteOut, testMappings)
+}
 
-	out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
+func fixEdges(rIn, rOut []rune, quoteOut []bool, edges []edge) {
-	for _, i := range []int{0, 1, testL - 2, testL - 1} {
+	testL := len(rIn)
-		for _, j := range []int{1, testL - 2, testL - 1} {
+	for _, e := range edges {
-			if j < i {
+		for idx, o := range e.orig {
-				continue
+			r := e.replace[idx]
-			}
+			if e.edge == edgeLeft && rIn[0] == o {
-			rIn := append([]rune{}, rIn...)
+				rOut[0], quoteOut[0] = r, false
-			rOut := append([]rune{}, rOut...)
+			} else if e.edge == edgeLeft && rIn[0] == r {
-			quoteOut := append([]bool{}, quoteOut...)
+				quoteOut[0] = true
-
+			} else if e.edge == edgeRight && rIn[testL-1] == o {
-			for _, in := range []rune{orig, replace} {
+				rOut[testL-1], quoteOut[testL-1] = r, false
-				expect, quote := in, false
+			} else if e.edge == edgeRight && rIn[testL-1] == r {
-				if i == 0 && edge == edgeLeft ||
+				quoteOut[testL-1] = true
-					i == testL-1 && edge == edgeRight {
-					expect, quote = replace, in == replace
-				}
-				rIn[i], rOut[i], quoteOut[i] = in, expect, quote
-
-				if i != j {
-					for _, in := range []rune{orig, replace} {
-						expect, quote = in, false
-						if j == testL-1 && edge == edgeRight {
-							expect, quote = replace, in == replace
-						}
-						rIn[j], rOut[j], quoteOut[j] = in, expect, quote
-					}
-				}
-				out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
 			}
 		}
 	}
-	return
 }
 
 func runePos(r rune, s []rune) int {