Lichen

Annotated test_all.sh

464:d0eae4e1bdc3
2017-01-12 Paul Boddie Added SystemExit support.
paul@4 1
#!/bin/sh
paul@4 2
paul@456 3
# This tool runs the toolchain for each of the tests, optionally building and
paul@456 4
# running the test programs.
paul@456 5
#
paul@456 6
# Copyright (C) 2016 Paul Boddie <paul@boddie.org.uk>
paul@456 7
#
paul@456 8
# This program is free software; you can redistribute it and/or modify it under
paul@456 9
# the terms of the GNU General Public License as published by the Free Software
paul@456 10
# Foundation; either version 3 of the License, or (at your option) any later
paul@456 11
# version.
paul@456 12
#
paul@456 13
# This program is distributed in the hope that it will be useful, but WITHOUT
paul@456 14
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
paul@456 15
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
paul@456 16
# details.
paul@456 17
#
paul@456 18
# You should have received a copy of the GNU General Public License along with
paul@456 19
# this program.  If not, see <http://www.gnu.org/licenses/>.
paul@456 20
paul@456 21
PROGNAME=$0
paul@456 22
OPTION=$1
paul@456 23
paul@442 24
LPLC="./lplc"
paul@442 25
DATADIR="_lplc"
paul@456 26
TESTINPUT="_results/testinput.txt"
paul@442 27
paul@45 28
# Expect failure from the "bad" tests.
paul@45 29
paul@45 30
expect_failure() {
paul@447 31
    echo "$FILENAME" | grep -q '_bad[._/]'
paul@447 32
    return $?
paul@45 33
}
paul@45 34
paul@45 35
# Check deduction output for type warnings, indicating that the program contains
paul@45 36
# errors.
paul@45 37
paul@45 38
check_type_warnings() {
paul@45 39
paul@442 40
    if [ -e "$DATADIR/_deduced/type_warnings" ] && \
paul@442 41
       [ `stat -c %s "$DATADIR/_deduced/type_warnings"` -ne 0 ] ; then
paul@45 42
paul@45 43
       echo "Type warnings in deduced information." 1>&2
paul@45 44
       return 1
paul@45 45
    fi
paul@45 46
paul@45 47
    return 0
paul@45 48
}
paul@45 49
paul@45 50
# Main program.
paul@45 51
paul@456 52
# Show help if requested.
paul@456 53
paul@456 54
if [ "$OPTION" = '--help' ] ; then
paul@456 55
    cat 1>&2 <<EOF
paul@456 56
Usage: $0 [ --build | --build-only | --run-built ]
paul@456 57
paul@456 58
Run the toolchain over all tests in the tests directory.
paul@323 59
paul@456 60
If --build is specified, the generated program code will be compiled and run,
paul@456 61
with the results collected in the _results directory.
paul@456 62
paul@456 63
If --build-only is specified, the generated programs themselves will be
paul@456 64
collected in the _results directory, each taking their name from that of the
paul@456 65
main program file used as input.
paul@456 66
paul@456 67
If --run-built is specified, any generated programs in the _results directory
paul@456 68
will be run and their output collected in the _results directory.
paul@323 69
paul@456 70
By using --build-only on one system, copying the _results directory to another
paul@456 71
system, and then running this script with the --run-built option on the other
paul@456 72
system, it becomes possible to test the toolchain output on the other system
paul@456 73
without needing to actually use the toolchain on that system. This permits the
paul@456 74
testing of cross-compiled programs.
paul@456 75
paul@456 76
For example, to build on one system but not run the generated programs:
paul@456 77
paul@456 78
ARCH=mipsel-linux-gnu $PROGNAME --build-only
paul@455 79
paul@456 80
And to run the generated programs on another system:
paul@456 81
paul@456 82
$PROGNAME --run-built
paul@456 83
paul@456 84
Of course, this script will need to be copied to the target system if it is not
paul@456 85
already available there.
paul@456 86
paul@456 87
Build and output logs are stored in the _results directory with the .build and
paul@456 88
.out suffixes employed respectively, one of each kind for each generated
paul@456 89
program.
paul@456 90
EOF
paul@456 91
    exit 1
paul@323 92
fi
paul@323 93
paul@455 94
# If just running existing programs, do so now and exit.
paul@455 95
paul@455 96
if [ "$OPTION" = '--run-built' ] ; then
paul@455 97
    for FILENAME in _results/* ; do
paul@455 98
        TESTNAME=`basename "$FILENAME"`
paul@455 99
paul@455 100
        # Skip non-program files.
paul@455 101
paul@455 102
        if [ ! -x "$FILENAME" ]; then
paul@455 103
            continue
paul@455 104
        fi
paul@455 105
paul@455 106
        echo "$FILENAME..." 1>&2
paul@455 107
        OUTLOG="_results/$TESTNAME.out"
paul@455 108
paul@455 109
        echo " (run)..." 1>&2
paul@455 110
        if ! "$FILENAME" > "$OUTLOG" < "$TESTINPUT" ; then
paul@455 111
            exit 1
paul@455 112
        fi
paul@455 113
    done
paul@455 114
paul@455 115
    exit 0
paul@455 116
fi
paul@331 117
paul@456 118
# Make any required results directory.
paul@456 119
paul@456 120
if [ "$OPTION" = '--build' ] || [ "$OPTION" = '--build-only' ] ; then
paul@456 121
    if [ ! -e "_results" ]; then
paul@456 122
        mkdir "_results"
paul@456 123
    else
paul@456 124
        rm "_results/"*
paul@456 125
    fi
paul@456 126
paul@456 127
    cp "tests/testinput.txt" "$TESTINPUT"
paul@456 128
fi
paul@456 129
paul@323 130
# Perform each test.
paul@323 131
paul@4 132
for FILENAME in tests/* ; do
paul@323 133
    TESTNAME=`basename "$FILENAME" .py`
paul@4 134
paul@4 135
    # Detect tests in their own subdirectories.
paul@4 136
paul@4 137
    if [ -d "$FILENAME" ] ; then
paul@4 138
        if [ -e "$FILENAME/main.py" ] ; then
paul@4 139
            FILENAME="$FILENAME/main.py"
paul@4 140
        else
paul@4 141
            continue
paul@4 142
        fi
paul@4 143
    fi
paul@4 144
paul@331 145
    # Skip non-program files.
paul@331 146
paul@331 147
    if [ `basename "$FILENAME"` = "$TESTNAME" ]; then
paul@331 148
        continue
paul@331 149
    fi
paul@331 150
paul@4 151
    # Run tests without an existing cache.
paul@4 152
paul@4 153
    echo "$FILENAME..." 1>&2
paul@442 154
    if ! "$LPLC" -c -r "$FILENAME" ; then
paul@447 155
        if ! expect_failure ; then
paul@275 156
            exit 1
paul@275 157
        else
paul@275 158
            echo 1>&2
paul@275 159
            continue
paul@275 160
        fi
paul@275 161
    fi
paul@39 162
paul@39 163
    # Check for unresolved names in the cache.
paul@39 164
paul@39 165
    echo " (depends)..." 1>&2
paul@442 166
    for CACHEFILE in "$DATADIR/_cache/"* ; do
paul@393 167
        STARTLINE=`grep -n '^deferred:' "$CACHEFILE" | cut -d: -f 1`
paul@393 168
        if tail -n +$(($STARTLINE + 2)) "$CACHEFILE" | grep -q '<depends>' ; then
paul@393 169
           echo "Unresolved names in the cache." 1>&2
paul@393 170
           exit 1
paul@393 171
        fi
paul@393 172
    done
paul@4 173
paul@45 174
    # Check for type warnings in deduction output.
paul@45 175
paul@45 176
    echo " (warnings)..." 1>&2
paul@45 177
    if ! check_type_warnings ; then exit 1 ; fi
paul@45 178
paul@4 179
    # Run tests with an existing cache.
paul@4 180
paul@39 181
    echo " (cached)..." 1>&2
paul@442 182
    if ! "$LPLC" -c "$FILENAME" ; then exit 1 ; fi
paul@4 183
paul@45 184
    echo " (warnings)..." 1>&2
paul@45 185
    if ! check_type_warnings ; then exit 1 ; fi
paul@45 186
paul@323 187
    # Build and run if appropriate.
paul@323 188
paul@455 189
    if [ "$OPTION" = '--build' ] || [ "$OPTION" = "--build-only" ] ; then
paul@323 190
        BUILDLOG="_results/$TESTNAME.build"
paul@323 191
        OUTLOG="_results/$TESTNAME.out"
paul@323 192
paul@323 193
        echo " (build)..." 1>&2
paul@442 194
        if ! make -C "$DATADIR/_generated" clean > "$BUILDLOG" || \
paul@442 195
           ! make -C "$DATADIR/_generated" > "$BUILDLOG" ; then
paul@323 196
            exit 1
paul@323 197
        fi
paul@323 198
paul@455 199
        if [ "$OPTION" = "--build-only" ]; then
paul@455 200
            mv "$DATADIR/_generated/main" "_results/$TESTNAME"
paul@455 201
        else
paul@455 202
            echo " (run)..." 1>&2
paul@455 203
            if ! "$DATADIR/_generated/main" > "$OUTLOG" < "$TESTINPUT" ; then
paul@455 204
                exit 1
paul@455 205
            fi
paul@323 206
        fi
paul@323 207
    fi
paul@323 208
paul@45 209
    echo 1>&2
paul@4 210
done