Your P/invokes are definitely defined wrong. You can't name the files VTFlib.dll.[x86/x64]. You need VTFlib.[x86/x64].dll, and then you need to implement appropriate P/Invoke dispatchers ("trampolines").
For example, from Paint.NET: (please excuse the indentation)
(the "Processor" class is in PaintDotNet.SystemLayer, and is appropriate for plugins to use in this case)
using namespace PaintDotNet.SystemLayer;
internal static class PdnNativeMethods
{
// trampoline method to the "real" p/invokes -- this is the method that outside code uses
internal static unsafe void MemCopy(void* dst, void* src, UIntPtr bytes)
{
if (Processor.Architecture == ProcessorArchitecture.X64)
{
x64.MemCopy(dst, src, bytes);
}
else
{
x86.MemCopy(dst, src, bytes);
}
}
[suppressUnmanagedCodeSecurity]
private static class x64
{
[DllImport("PaintDotNet.Native.x64.dll", CallingConvention = CallingConvention.Cdecl, SetLastError = false)]
internal static extern unsafe void MemSet(void* dst, int c, UIntPtr length);
}
[suppressUnmanagedCodeSecurity]
private static class x86
{
[DllImport("PaintDotNet.Native.x86.dll", CallingConvention = CallingConvention.Cdecl, SetLastError = false)]
internal static extern unsafe void MemSet(void* dst, int c, UIntPtr length);
}