CC-tea/org-vim
1
0
Fork 0
mirror of https://github.com/vim/vim.git synced 2026-05-22 18:22:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

Revert "CI: Manage multibyte characters in syntax tests"

Browse Source 
This reverts commit 0fde6aebdd.

With "v9.1.2134" applied, arbitrary multibyte characters are
not replaced with spurious U+FFFD characters (typically when
the host system is under load); U+FFFD characters that are
intentionally written in an input file continue to remain
present for later output and comparison.  The workaround of
"v9.1.1592~3" is no longer necessary.

Also prefer "page_nr" to "nr" in syntax/testdir/runtest.vim

closes: #19348

Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
This commit is contained in:
Aliaksei Budavei
2026-02-07 15:48:33 +00:00
committed by Christian Brabandt
parent 35c428b02c
commit f0b8b2c32a
4 changed files with 69 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
runtime/syntax/Makefile
View File

@@ -13,7 +13,7 @@ VIMRUNTIME = ../..
# Uncomment this line to use valgrind for memory leaks and extra warnings.
# VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=45 --log-file=valgrind.$*
# Trace liveness on demand.
# Trace ruler liveness on demand.
# VIM_SYNTAX_TEST_LOG = `pwd`/testdir/failed/00-TRACE_LOG
# ENVVARS = LC_ALL=C VIM_SYNTAX_TEST_LOG="$(VIM_SYNTAX_TEST_LOG)"
@@ -39,7 +39,7 @@ test:
@# the "vimcmd" file is used by the screendump utils
@echo "../$(VIMPROG)" > testdir/vimcmd
@echo "$(RUN_VIMTEST)" >> testdir/vimcmd
@# Trace liveness on demand.
@# Trace ruler liveness on demand.
@#mkdir -p testdir/failed
@#touch "$(VIM_SYNTAX_TEST_LOG)"
VIMRUNTIME=$(VIMRUNTIME) $(ENVVARS) $(VIMPROG) --clean --not-a-term $(DEBUGLOG) -u testdir/runtest.vim > /dev/null

16
runtime/syntax/testdir/README.txt
View File

@@ -61,6 +61,8 @@ an "input/setup/java.vim" script file with the following lines:
Both inline setup commands and setup scripts may be used at the same time, the
script file will be sourced before any VIM_TEST_SETUP commands are executed.
Every line of a source file must not be longer than 1425 (19 x 75) characters.
If there is no further setup required, you can now run all tests:
make test
@@ -110,20 +112,6 @@ If they look OK, move them to the "dumps" directory:
If you now run the test again, it will succeed.
Limitations for syntax plugin tests
-----------------------------------
Do not compose ASCII lines that do not fit a 19 by 75 window (1425 columns).
Use multibyte characters, if at all, sparingly (see #16559). When possible,
move multibyte characters closer to the end of a line and keep the line short:
no more than a 75-byte total of displayed characters. A poorly rendered line
may otherwise become wrapped when enough of spurious U+FFFD (0xEF 0xBF 0xBD)
characters claim more columns than are available (75) and then invalidate line
correspondence under test. Refrain from mixing non-spurious U+FFFD characters
with other multibyte characters in the same line.
Adjusting a syntax plugin test
------------------------------

309
runtime/syntax/testdir/runtest.vim
View File

@@ -116,54 +116,15 @@ func HandleSwapExists()
endif
endfunc
" Trace ruler liveness on demand.
if !empty($VIM_SYNTAX_TEST_LOG) && filewritable($VIM_SYNTAX_TEST_LOG)
" Trace liveness.
def s:TraceLiveness(context: string, times: number, tail: string)
def s:TraceRulerLiveness(context: string, times: number, tail: string)
writefile([printf('%s: %4d: %s', context, times, tail)],
$VIM_SYNTAX_TEST_LOG,
'a')
enddef
" Anticipate rendering idiosyncrasies (see #16559).
def s:CanFindRenderedFFFDChars(
buf: number,
in_name_and_out_name: string,
times: number): bool
if CannotUseRealEstate(in_name_and_out_name)
return false
endif
# Expect a 20-line terminal buffer (see "term_util#RunVimInTerminal()"),
# where the bottom, reserved line is of the default "&cmdheight".
var lines: list<number> = []
for lnum: number in range(1, 19)
if stridx(term_getline(buf, lnum), "\xef\xbf\xbd") >= 0
add(lines, lnum)
endif
endfor
TraceLiveness('F', times, string(lines))
return !empty(lines)
enddef
else
" Do not trace liveness.
def s:TraceLiveness(_: string, _: number, _: string)
enddef
" Anticipate rendering idiosyncrasies (see #16559).
def s:CanFindRenderedFFFDChars(
buf: number,
in_name_and_out_name: string,
_: number): bool
if CannotUseRealEstate(in_name_and_out_name)
return false
endif
# Expect a 20-line terminal buffer (see "term_util#RunVimInTerminal()"),
# where the bottom, reserved line is of the default "&cmdheight".
for lnum: number in range(1, 19)
if stridx(term_getline(buf, lnum), "\xef\xbf\xbd") >= 0
return true
endif
endfor
return false
def s:TraceRulerLiveness(_: string, _: number, _: string)
enddef
endif
@@ -191,68 +152,6 @@ def s:CannotDumpShellFirstPage(buf: number, _: list<string>, ruler: list<string>
get(term_getcursor(buf), 0) != 20)
enddef
def s:CannotUseRealEstate(in_name_and_out_name: string): bool
# Expect defaults from "term_util#RunVimInTerminal()".
if winwidth(1) != 75 || winheight(1) != 20
ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
in_name_and_out_name,
winwidth(1),
winheight(1)))
return true
endif
return false
enddef
" Throw an "FFFD" string if U+FFFD characters are found in the terminal buffer
" during a non-last test round; otherwise, generate a screendump and proceed
" with its verification.
def s:VerifyScreenDumpOrThrowFFFD(
buf: number,
which_page: string,
in_name_and_out_name: string,
aborted_count: number,
max_aborted_count: number,
basename: string,
opts: dict<any>,
page_quota: dict<number>,
seen_pages: list<number>,
page_nr: number): number
if !has_key(page_quota, page_nr)
# Constrain management of unseen pages to the product of "wait" times
# "max_aborted_count" (see "opts" below). When _test repetition_ and
# _line rewriting_ FAIL page verification, the page gets to keep its
# unseen mark; when _test repetition_ is FAILING for a later page, all
# earlier unseen pages get another chance at _test repetition_ etc. before
# further progress can be made for the later page.
page_quota[page_nr] = max_aborted_count
endif
const with_fffd: bool = CanFindRenderedFFFDChars(
buf,
in_name_and_out_name,
(max_aborted_count - aborted_count + 1))
if with_fffd && aborted_count > 1
throw 'FFFD'
endif
ch_log(which_page .. ' screendump for ' .. in_name_and_out_name)
# Generate a screendump of every 19 lines of "buf", reusing the bottom line
# (or the bottom six or so lines for "*_01.dump") from the previous dump as
# the top line(s) in the next dump for continuity. Constrain generation of
# unseen pages for the last test round (via "wait").
const status: number = g:VerifyScreenDump(
buf,
basename,
(aborted_count != max_aborted_count)
? extend({wait: max_aborted_count}, opts, 'keep')
: opts)
if !with_fffd || (!status || !page_quota[page_nr])
add(seen_pages, page_nr)
else
TraceLiveness('Q', (max_aborted_count - aborted_count + 1), string(page_quota))
endif
page_quota[page_nr] -= 1
return status
enddef
" Poll for updates of the cursor position in the terminal buffer occupying the
" first window. (ALWAYS call the function or its equivalent before calling
" "VerifyScreenDump()" *and* after calling any number of "term_sendkeys()".)
@@ -260,7 +159,12 @@ def s:TermPollRuler(
CannotDumpPage: func, # (TYPE FOR LEGACY CONTEXT CALL SITES.)
buf: number,
in_name_and_out_name: string): list<string>
if CannotUseRealEstate(in_name_and_out_name)
# Expect defaults from "term_util#RunVimInTerminal()".
if winwidth(1) != 75 || winheight(1) != 20
ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
in_name_and_out_name,
winwidth(1),
winheight(1)))
return ['0,0-1', 'All']
endif
# A two-fold role for redrawing:
@@ -283,7 +187,7 @@ def s:TermPollRuler(
redraw
endif
endwhile
TraceLiveness('P', (2048 - times), in_name_and_out_name)
TraceRulerLiveness('P', (2048 - times), in_name_and_out_name)
return ruler
enddef
@@ -292,7 +196,12 @@ enddef
" the terminal buffer. (Call the function before calling "VerifyScreenDump()"
" for the first time.)
def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list<string>
if CannotUseRealEstate(in_name_and_out_name)
# Expect defaults from "term_util#RunVimInTerminal()".
if winwidth(1) != 75 || winheight(1) != 20
ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
in_name_and_out_name,
winwidth(1),
winheight(1)))
return ['0,0-1', 'All']
endif
# The contents of "ruler".
@@ -311,7 +220,7 @@ def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list<stri
redraw
endif
endwhile
TraceLiveness('W', (32768 - times), in_name_and_out_name)
TraceRulerLiveness('W', (32768 - times), in_name_and_out_name)
if strpart(ruler, 0, 8) !=# 'is_nonce'
# Retain any of "b:is_(bash|dash|kornshell|posix|sh)" entries and let
# "CannotDumpShellFirstPage()" win the cursor race.
@@ -384,34 +293,6 @@ func RunTest()
redraw!
endfunc
def CollectFFFDChars()
const fffd: string = "\xef\xbf\xbd"
const flags: string = 'eW'
const pos: list<number> = getpos('.')
var fffds: list<list<number>> = []
try
cursor(1, 1)
var prev: list<number> = [0, 0]
var next: list<number> = [0, 0]
next = searchpos(fffd, 'c' .. flags)
while next[0] > 0 && prev != next
add(fffds, next)
prev = next
next = searchpos(fffd, flags)
endwhile
finally
setpos('.', pos)
endtry
if !empty(fffds)
# Use "actions/upload-artifact@v4" of ci.yml for delivery.
writefile(
[printf('%s: %s', bufname('%'), string(fffds))],
'failed/10-FFFDS',
'a')
endif
redraw!
enddef
def s:AssertCursorForwardProgress(): bool
const curnum: number = line('.')
if curnum <= cursor
@@ -512,18 +393,12 @@ func RunTest()
return AssertCursorForwardProgress()
enddef
END
let MAX_ABORTED_COUNT = 5
let MAX_FAILED_COUNT = 5
let DUMP_OPTS = extend(
\ exists("$VIM_SYNTAX_TEST_WAIT_TIME") &&
let DUMP_OPTS = exists("$VIM_SYNTAX_TEST_WAIT_TIME") &&
\ !empty($VIM_SYNTAX_TEST_WAIT_TIME)
\ ? {'wait': max([1, str2nr($VIM_SYNTAX_TEST_WAIT_TIME)])}
\ : {},
\ {'FileComparisonPreAction':
\ function('g:ScreenDumpDiscardFFFDChars'),
\ 'NonEqualLineComparisonPostAction':
\ function('g:ScreenDumpLookForFFFDChars')})
lockvar DUMP_OPTS MAX_FAILED_COUNT MAX_ABORTED_COUNT XTESTSCRIPT
\ ? {'wait': max([1, str2nr($VIM_SYNTAX_TEST_WAIT_TIME)])}
\ : {}
lockvar DUMP_OPTS MAX_FAILED_COUNT XTESTSCRIPT
let ok_count = 0
let disused_pages = []
let failed_tests = []
@@ -591,107 +466,53 @@ func RunTest()
let args = !empty(path)
\ ? prefix .. ' -S ' .. path
\ : prefix
let fail = 0
let buf = RunVimInTerminal(args, {})
" edit the file only after catching the SwapExists event
call term_sendkeys(buf, ":edit " .. fname .. "\<CR>")
" set up the testing environment
call term_sendkeys(buf, ":call SetUpVim()\<CR>")
" load filetype specific settings
call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\<CR>")
" Make a synchronisation point between buffers by requesting to echo
" a known token in the terminal buffer and asserting its availability
" with "s:TermWaitAndPollRuler()".
if filetype == 'sh'
call term_sendkeys(buf, ":call ShellInfo()\<CR>")
else
call term_sendkeys(buf, ":echo 'is_nonce'\<CR>")
endif
let root_00 = root .. '_00'
let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump'
" Queue up all "term_sendkeys()"es and let them finish before returning
" from "s:TermWaitAndPollRuler()".
let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name)
call ch_log('First screendump for ' .. in_name_and_out_name)
" Make a screendump at the start of the file: failed/root_00.dump
let fail = VerifyScreenDump(buf, root_00, DUMP_OPTS)
let page_nr = 0
" Accommodate the next code block to "buf"'s contingency for self
" wipe-out.
try
let aborted_count = MAX_ABORTED_COUNT
let collected_count = 0
let seen_pages = []
let page_quota = {}
" See #16559. For each processed page, repeat pre-verification steps
" from scratch (subject to page cacheing) whenever U+FFFD characters
" are found in the terminal buffer with "term_getline()", i.e. treat
" these pages as if they were distinct test files. U+FFFD characters
" found at the last attempt (see "MAX_ABORTED_COUNT") will be ignored
" and "VerifyScreenDump()" will take over with own filtering.
while aborted_count > 0
let buf = RunVimInTerminal(args, {})
try
" edit the file only after catching the SwapExists event
call term_sendkeys(buf, ":edit " .. fname .. "\<CR>")
" set up the testing environment
call term_sendkeys(buf, ":call SetUpVim()\<CR>")
" load filetype specific settings
call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\<CR>")
" Collect all *non-spurious* U+FFFD characters for scrutiny.
if aborted_count == 1 && collected_count != 1
let collected_count = 1
call term_sendkeys(buf, ":call CollectFFFDChars()\<CR>")
endif
" Make a synchronisation point between a terminal buffer and
" another buffer by requesting to echo a known token in the former
" and asserting its availability with "s:TermWaitAndPollRuler()"
" from the latter.
if filetype == 'sh'
call term_sendkeys(buf, ":call ShellInfo()\<CR>")
else
call term_sendkeys(buf, ":echo 'is_nonce'\<CR>")
endif
let page_nr = 0
let root_00 = root .. '_00'
let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump'
" Queue up all "term_sendkeys()"es and let them finish before
" returning from "s:TermWaitAndPollRuler()".
let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name)
if index(seen_pages, page_nr) < 0
let fail += s:VerifyScreenDumpOrThrowFFFD(
\ buf,
\ 'First',
\ in_name_and_out_name,
\ aborted_count,
\ MAX_ABORTED_COUNT,
\ root_00,
\ DUMP_OPTS,
\ page_quota,
\ seen_pages,
\ page_nr)
" Reset "aborted_count" for another page.
let aborted_count = MAX_ABORTED_COUNT
endif
let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
while s:CannotSeeLastLine(ruler)
call term_sendkeys(buf, keys_a)
let keys_a = keys_b
let page_nr += 1
let root_next = printf('%s_%02d', root, page_nr)
let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump'
let ruler = s:TermPollRuler(
\ function('s:CannotDumpNextPage', [buf, ruler]),
\ buf,
\ in_name_and_out_name)
if index(seen_pages, page_nr) < 0
let fail += s:VerifyScreenDumpOrThrowFFFD(
\ buf,
\ 'Next',
\ in_name_and_out_name,
\ aborted_count,
\ MAX_ABORTED_COUNT,
\ root_next,
\ DUMP_OPTS,
\ page_quota,
\ seen_pages,
\ page_nr)
" Reset "aborted_count" for another page.
let aborted_count = MAX_ABORTED_COUNT
endif
endwhile
call StopVimInTerminal(buf)
break
catch /^FFFD$/
" Clear out.
call StopVimInTerminal(buf)
while winnr('$') > 1
close
endwhile
let aborted_count -= 1
endtry
let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
while s:CannotSeeLastLine(ruler)
call term_sendkeys(buf, keys_a)
let keys_a = keys_b
let page_nr += 1
let root_next = printf('%s_%02d', root, page_nr)
let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump'
let ruler = s:TermPollRuler(
\ function('s:CannotDumpNextPage', [buf, ruler]),
\ buf,
\ in_name_and_out_name)
call ch_log('Next screendump for ' .. in_name_and_out_name)
" Make a screendump of every 18 lines of the file: failed/root_NN.dump
let fail += VerifyScreenDump(buf, root_next, DUMP_OPTS)
endwhile
call StopVimInTerminal(buf)
finally
call delete('Xtestscript')
endtry