-------------------------------------
I think, this is the wrong declaration
You'll better try Dim retc as INTEGER

Function IsInst(hInst As Long) As Boolean

    Dim taskstruct As TaskEntry

    Dim retc as integer
    IsInst = False
    taskstruct.dwSize = Len(taskstruct)
    retc = TaskFirst(taskstruct)

    Do While retc <> 0

      If taskstruct.hInst = hInst Then
            ' note: the task handle is: taskstruct.hTask
            IsInst = True
            Exit Function
        End If
        retc = TaskNext(taskstruct)
    Loop

End Function

The results returned by the functions TaskFirst and TaskNext are
integer as well.

I use the following declaration:

Private Type TASKENTRY ' 40 bytes
    dwSize As Long
    hTask As Integer
    hTaskParent As Integer
    hInst As Integer
    hModule As Integer
    wSS As Integer
    wSP As Integer
    wStackTop As Integer
    wStackMinimum As Integer
    wStackBottom As Integer
    wcEvents As Integer
    hQueue As Integer
    szModule As String * 10
    wPSPOffSet As Integer
    hNext As Integer
End Type

Private Declare Function TaskFirst Lib "toolhelp.dll" (taskstruct As
TASKENTRY) as integer
Private Declare Function TaskNext Lib "toolhelp.dll" (taskstruct As
TASKENTRY) as integer