The syntax error is easily fixed by
...OpenFile(PAnsiChar(FileName), ..
but it doesn't actually work.
This one does.
function FileInUse(FileName: string): Boolean;
var hFileRes: HFILE;
begin
Result := False;
if not FileExists(FileName) then exit;
hFileRes := CreateFile(PChar(FileName),
GENERIC_READ or GENERIC_WRITE,
0,
nil,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
0);
Result := (hFileRes = INVALID_HANDLE_VALUE);
if not Result then
CloseHandle(hFileRes);
end;