DosTips.com

Code: Select all

>type example-utf8-with-bom.txt
я╗┐In math the Greek letter ╧А stands for 3.1415926

>type example-utf8-without-bom.txt
In math the Greek letter ╧А stands for 3.1415926

>file-detect-enc.bat example-utf8-*.txt
example-utf8-with-bom.txt: UTF-8
example-utf8-without-bom.txt: UTF-8

Code: Select all

::Usage: file-detect-enc [OPTIONS] FILE...
::
::Detect type (encoding) of FILEs.
::
::  -b, --brief  Don't prepend filenames to output

@echo off

setlocal enabledelayedexpansion

set "enc_brief="
if /i "%~1" == "-b"      set "enc_brief=1"
if /i "%~1" == "--brief" set "enc_brief=1"
if defined enc_brief shift /1

if "%~1" == "" goto :print_usage

:: ========================================================================

:: The following settings are based on information from the table
:: https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding
set "enc_val_EFBBBF=UTF-8"
set "enc_val_FEFF=UTF-16BE"
set "enc_val_FFFE=UTF-16LE"
set "enc_val_0000FEFF=UTF-32BE"
set "enc_val_FFFE0000=UTF-32LE"
set "enc_val_2B2F76=UTF-7"
set "enc_val_F7644C=UTF-1"
set "enc_val_DD736673=UTF-EBCDIC"
set "enc_val_0EFEFF=SCSU"
set "enc_val_FBEE28=BOCU-1"
set "enc_val_84319533=GB-18030"

:: ========================================================================

set "enc_hexfile=%TEMP%\enc_hexfile"

:enc_begin_loop
if "%~1" == "" (
	del /f /q "%enc_hexfile%" 2>nul
	goto :EOF
)

for %%f in ( "%~1" ) do (
	call :detect_type "%%~f"

	if defined enc_brief (
		if defined enc_found echo:!enc_found!
	) else (
		echo:%%~f: !enc_found!
	)
)

shift /1
goto :enc_begin_loop

:: ========================================================================

:print_usage
for /f "usebackq tokens=* delims=:" %%s in ( "%~f0" ) do (
	if /i "%%s" == "@echo off" goto :EOF
	echo:%%s
)
goto :EOF

:: ========================================================================

:detect_type
set "enc_found="

set "enc_srcfile=%~1"

if not exist "%~1" (
	echo:File not found: "%~1">&2
	exit /b 1
)
if exist "%~1\" (
	set "enc_found=directory"
	goto :EOF
)
if %~z1 equ 0 (
	set "enc_found=empty"
	goto :EOF
)

:: https://stackoverflow.com/a/16238102/3627676
:: https://ss64.com/nt/certutil.html
:: https://www.dostips.com/forum/viewtopic.php?p=57918#p57918
:: https://docs.microsoft.com/en-gb/windows/win32/api/wincrypt/nf-wincrypt-cryptbinarytostringa
certutil -encodehex -f "%enc_srcfile%" "%enc_hexfile%" 4 >nul || (
	echo:Internal error: !errorlevel!>&2
	exit /b 1
)

set "enc_utf8_sequence="
set /a enc_utf8_require=0

set "enc_firstline=1"
for /f "usebackq delims=" %%s in ( "%enc_hexfile%" ) do (
	rem Most files (especially binaries) have in their beginning the
	rem magic number, or header, the group of bytes identifying the
	rem file type. Here we can analyze the header for magic number
	rem existence and quit immediately, if it's found. Otherwise,
	rem we continue analysis with the same line.
	if defined enc_firstline for /f "usebackq tokens=1-4" %%a in ( '%%s' ) do (
		set "enc_firstline="
		set "enc_bytes=%%a%%b%%c%%d"

		for /l %%n in ( 8, -2, 4 ) do for %%s in (
			enc_val_!enc_bytes:~0^,%%n!
		) do (
			set "enc_found=!%%s!"
			if defined enc_found goto :EOF
		)
	)

	rem https://en.wikipedia.org/wiki/UTF-8#Encoding
	rem 0000-007f		00-7f	-----	-----	-----
	rem 0080-07ff		c0-df	80-bf	-----	-----
	rem 0800-ffff		e0-ef	80-bf	80-bf	-----
	rem 10000-10ffff	f0-f7	80-bf	80-bf	80-bf
	for %%b in ( %%s ) do if 0x%%b lss 0x80 (
		rem 00-7f
		set "enc_utf8_sequence="
		set /a enc_utf8_require=0
	) else if 0x%%b gtr 0xf7 (
		rem f8-ff
		set "enc_utf8_sequence="
		set /a enc_utf8_require=0
	) else (
		rem 80-f7
		set "enc_utf8_sequence=!enc_utf8_sequence!%%b"

		if 0x%%b geq 0xf0 (
			rem f0-f7
			set /a enc_utf8_require=3
		) else if 0x%%b geq 0xe0 (
			rem e0-ef
			set /a enc_utf8_require=2
		) else if 0x%%b geq 0xc0 (
			rem c0-df
			set /a enc_utf8_require=1
		) else if !enc_utf8_require! gtr 0 (
			rem 80-bf
			set /a enc_utf8_require-=1
			if !enc_utf8_require! equ 0 (
				set "enc_found=UTF-8"
				goto :EOF
			)
		)
	)
)
goto :EOF

:: ========================================================================

:: EOF

Code: Select all

:print_usage

Code: Select all

set "file=u8nobom.txt"

Code: Select all

set "file=%~f0"

Code: Select all

@if "%~1"=="::check::" (goto check) else echo off &setlocal DisableDelayedExpansion

set "file=u8nobom.txt"

set "me=%~fs0"
setlocal EnableDelayedExpansion
:: write all hex values in a row without spaces
certutil -encodehex -f "!file!" "!temp!\!file!.hex" 12 >nul || exit /b 1
:: read only up to 1023 characters (limit of set /p)
<"!temp!\!file!.hex" set /p "s="
del "!temp!\!file!.hex"
:: try to determine the encoding by a Byte Order Mark
set "bomFFFE0000=UTF-32 LE" &set "bom0000FEFF=UTF-32 BE" &set "bomDD736673=UTF-EBCDIC" &set "bom84319533=GB-18030"
set "bom2B2F7638=UTF-7"     &set "bom2B2F7639=UTF-7"     &set "bom2B2F762B=UTF-7"      &set "bom2B2F762F=UTF-7"
set "bomEFBBBF=UTF-8"       &set "bomF7644C=UTF-1"       &set "bom0EFEFF=SCSU"         &set "bomFBEE28=BOCU-1"
set "bomFFFE=UTF-16 LE"     &set "bomFEFF=UTF-16 BE"
set "enc="&for %%i in (8 6 4) do if not defined enc for %%j in (bom!s:~^,%%i!) do set "enc=!%%j!"
if defined enc (
  echo !enc!
  pause
  goto :eof
)

:: compute the length of s
set s=A%s%
set "lastX=0"
for /l %%i in (12 -1 0) do (
  set /a "lastX|=1<<%%i"
  for %%j in (!lastX!) do if "!s:~%%j,1!"=="" set /a "lastX&=~1<<%%i"
)
:: last index is the length (rounded to the greatest even number that is not greater than the length), minus 7 (because we want to read 8 hex chars at a time, but 1 char is appended for length measuring)
set /a "lastX=(lastX/2)*2-7"
if %lastX% lss 1 (
  REM maybe work around strings with less than 4 bytes here
  goto :eof
)
:: perform the UTF-8 check
"%comspec%" /q /d /von /c "!me!" ::check::
if errorlevel 2 (        REM errorlevel 2..7
  echo UTF-8 (w\ multibyte sequences^)
) else if errorlevel 1 ( REM errorlevel exactly 1
  echo ASCII only
) else (                 REM errorlevel 0
  echo no UTF-8
)
pause
goto :eof

:check
:: Lengths array of multibyte sequences, based on https://github.com/skeeto/branchless-utf8/blob/master/utf8.h, but bytes 0x00..0x07 and 0xF8..0xFF are explicitly treated as invalid
set /a "L0=8,L1=1,L2=1,L3=1,L4=1,L5=1,L6=1,L7=1,L8=1,L9=1,L10=1,L11=1,L12=1,L13=1,L14=1,L15=1,L16=0,L17=0,L18=0,L19=0,L20=0,L21=0,L22=0,L23=0,L24=2,L25=2,L26=2,L27=2,L28=3,L29=3,L30=4,L31=8"
:: Shift array used to shift out lengths that don't belong to the expected continuation bytes, initialize return value
:: x1..x4 are the indexes of the bytes we want to read in the hex string (begins with 1 because an "A" is prepended for length measuring)
set /a "Sh0=0,Sh1=9,Sh2=6,Sh3=3,Sh4=0,ret=0,x1=1,x2=3,x3=5,x4=7"
for /l %%i in () do (
  REM indexes in the lengths array
  for /f "tokens=1-4" %%j in ("!x1! !x2! !x3! !x4!") do set /a "i1=0x!s:~%%j,2!>>3,i2=0x!s:~%%k,2!>>3,i3=0x!s:~%%l,2!>>3,i4=0x!s:~%%m,2!>>3"
  REM lengths, new indexes in the hex string, and updated return value
  set /a "n1=L!i1!,n2=L!i2!,n3=L!i3!,n4=L!i4!,x1+=n1*2,x2=x1+2,x3=x1+4,x4=x1+6,ret|=n1"
  REM the length of the first byte must not be 0, while the lengths of all continuation bytes have to be all 0, also no byte shall be less than 0x08 or greater than 0x0F7
  set /a "chk=(^!n1)|(((n2<<6)|(n3<<3)|n4)>>Sh!n1!)|((n1|n2|n3|n4)>>3)"
  if !chk! neq 0 exit 0
  if !x1! gtr !lastX! exit !ret!
)

Code: Select all

findstr /r "[\x80-\xf8]" FILENAME

Code: Select all

findstr /r "[^\x01-\x7f\xf8-\xff]" FILENAME

Code: Select all

type example-utf8-without-bom.txt | hexdump -C