win64/pe: tls callback assembly improvements + misc cleanups

NEWS

@@ -4,6 +4,7 @@ User visible changes for UPX
 
 Changes in 3.10 (XX XXX 2013):
   * Experimental support added for Windows 64-bit PE files.
+    based on the work by Stefan Widmann
   * bug fixes
 
 Changes in 3.09 (18 Feb 2013):

src/p_w64pep.cpp

@@ -1190,25 +1190,6 @@ void PackW64Pep::pack(OutputFile *fo)
     ODADDR(PEDIR_BOUNDIM) = 0;  //remove bound import table
     ODSIZE(PEDIR_BOUNDIM) = 0;
 
-    //EXCEPTION DIRECTORY HANDLING - Stefan Widmann
-    //APPROACH 1: just keep the exception directory, it's only used during runtime, not during init
-    // -> nothing to do
-#if 0
-    ODADDR(PEDIR_EXCEPTION) = 0;
-    ODSIZE(PEDIR_EXCEPTION) = 0;
-#endif
-#if 0
-    //APPROACH 2: we remove the exception directory from the header, the stub installs the table
-    //            after decompression by calling RtlAddFunctionTable (see MSDN for details)
-    ODADDR(PEDIR_EXCEPTION) = 0;
-    ODSIZE(PEDIR_EXCEPTION) = 0;
-    //set flag
-    use_exception_dir = true;
-    //link code to loader
-    linker->defineSymbol("exception_ptr", IDADDR(PEDIRE_EXCEPTION)); //stub needs data about function table
-    linker->defineSymbol("exception_size", IDSIZE(PEDIR_EXCEPTION));
-#endif
-
     // tls & loadconf are put into section 1
 
     //ic = s1addr + s1size - sotls - soloadconf;  //ATTENTION: moved upwards to TLS callback handling - Stefan Widmann

src/stub/src/amd64-win64.pep.S

@@ -302,7 +302,7 @@ section         PEIERDLL
 
 section         PEIEREXE
                 //      rcx contains garbage -> garbage return code
-                call    [rsi + ExitProcess]
+                jmp     [rsi + ExitProcess]
 section         PEIMDONE
 imports_done:
                 add     rsp, 0x28
@@ -407,18 +407,14 @@ section         PEDEPHAK
 // =============
 
 section         PETLSC
-                lea     rcx, [rsi + tls_module_base] //;load module base to rcx
+                movb    [rip + PETLSC2], 0xfc   // "cld" instead of "ret"
-                lea     rdi, [rcx + tls_handler_start + 1] //;load offset of handler
+                lea     rcx, [rsi + tls_module_base] // module base
-                //;remove jump from TLS handler entry (overwrite displacement)
+                push    1                       // DLL_PROCESS_ATTACH
-                xor     eax, eax
-                stosb
-                //;emulate callbacks like PE loader would have done
-                mov     r8, rax                 //;0 - reserved
-                push    1                       //;DLL_PROCESS_ATTACH
                 pop     rdx
+                xor     r8, r8                  // 0 - reserved
 
-                push    rax                     //;align stack
+                push    rax                     // align stack
-                call    rdi                     //;contains ptr to callback handler
+                call    PETLSC2
                 pop     rax
 
 // ============= Cleanup
@@ -462,13 +458,13 @@ section         PEDOJUMP
 // ============= TLS callback support part 2
 // =============
 
-section         PETLSC2
+// this is the new TLS callback handler
-                //;TLS_CALLBACK(hModule, reason, reserved)
+// it calls the original callbacks ONLY after the compression is done
-tls_handler_start:
+
-                jmp     end_of_tls_handler      //;this jump is patched to EB 00 (jmp $+2) by stub
+section         PETLSC2         // TLS_CALLBACK(hModule, reason, reserved)
+                ret             // this ret gets overwritten with cld by PETLSC
                 push    rsi
                 lea     rsi, [rip + tls_callbacks_ptr]
-                cld                             //;you never know, this code gets called by the PE loader
 walk_tlsc_chain2:
                 lodsq
                 test    rax, rax
@@ -489,7 +485,6 @@ walk_tlsc_chain2:
                 jmp     walk_tlsc_chain2
 done_callbacks:
                 pop     rsi
-end_of_tls_handler:
                 ret
 
 // =============

src/stub/tmp/amd64-win64.pep.bin.dump

@@ -50,8 +50,8 @@ Idx Name          Size      VMA               LMA               File off  Algn
  45 PEISDLL9      0000000f  0000000000000000  0000000000000000  000018f0  2**0  CONTENTS, READONLY
  46 PERETURN      00000004  0000000000000000  0000000000000000  000018ff  2**0  CONTENTS, READONLY
  47 PEDOJUMP      00000005  0000000000000000  0000000000000000  00001903  2**0  CONTENTS, RELOC, READONLY
- 48 PETLSC2       00000028  0000000000000000  0000000000000000  00001908  2**0  CONTENTS, RELOC, READONLY
+ 48 PETLSC2       00000026  0000000000000000  0000000000000000  00001908  2**0  CONTENTS, RELOC, READONLY
- 49 UPX1HEAD      00000020  0000000000000000  0000000000000000  00001930  2**0  CONTENTS, READONLY
+ 49 UPX1HEAD      00000020  0000000000000000  0000000000000000  0000192e  2**0  CONTENTS, READONLY
 SYMBOL TABLE:
 0000000000000000 l    d  NRV_HEAD	0000000000000000 NRV_HEAD
 0000000000000000 l    d  PEMAIN10	0000000000000000 PEMAIN10
@@ -62,8 +62,8 @@ SYMBOL TABLE:
 000000000000000a l       PEIMPOR2	0000000000000000 first_imp
 0000000000000002 l       RELOC64J	0000000000000000 reloc_endx
 0000000000000013 l       PERELOC3	0000000000000000 reloc_add
-0000000000000000 l    d  PEMAIN21	0000000000000000 PEMAIN21
 0000000000000000 l    d  PETLSC2	0000000000000000 PETLSC2
+0000000000000000 l    d  PEMAIN21	0000000000000000 PEMAIN21
 0000000000000000 l    d  START	0000000000000000 START
 0000000000000000 l    d  PEISDLL0	0000000000000000 PEISDLL0
 0000000000000000 l    d  PEISDLL1	0000000000000000 PEISDLL1
@@ -247,8 +247,9 @@ OFFSET           TYPE              VALUE
 
 RELOCATION RECORDS FOR [PETLSC]:
 OFFSET           TYPE              VALUE
-0000000000000003 R_X86_64_32S      tls_module_base
+0000000000000002 R_X86_64_PC32     PETLSC2+0xfffffffffffffffb
-000000000000000a R_X86_64_32S      PETLSC2+0x0000000000000001
+000000000000000a R_X86_64_32S      tls_module_base
+0000000000000016 R_X86_64_PC32     PETLSC2+0xfffffffffffffffc
 
 RELOCATION RECORDS FOR [PEDOJUMP]:
 OFFSET           TYPE              VALUE
@@ -256,4 +257,4 @@ OFFSET           TYPE              VALUE
 
 RELOCATION RECORDS FOR [PETLSC2]:
 OFFSET           TYPE              VALUE
-0000000000000006 R_X86_64_PC32     tls_callbacks_ptr+0xfffffffffffffffc
+0000000000000005 R_X86_64_PC32     tls_callbacks_ptr+0xfffffffffffffffc