root/xdelta30r/xdelta3-main.h

/* xdelta 3 - delta compression tools and library
 * Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007,
 * Joshua P. MacDonald
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/* This is all the extra stuff you need for convenience to users in a command line
 * application.  It contains these major components:
 *
 * 1. VCDIFF tools
 * 2. external compression support (this is POSIX-specific).
 * 3. a general read/write loop that handles all of the Xdelta decode/encode/VCDIFF-print
 *    functions
 * 4. command-line interpreter
 * 5. an Xdelta application header which stores default filename, external compression settings
 * 6. output/error printing
 * 7. basic file support and OS interface
 */

/* TODO list:
 * 1. do exact gzip-like filename, stdout handling.  make a .vcdiff extension, refuse
 *    to encode to stdout without -cf, etc.
 * 2. Allow the user to add a comment string to the app header without disturbing the default
 *    behavior.
 * 3. "Source file must be seekable" is not actually true for encoding, given current
 *    behavior.  Allow non-seekable sources?  It would in theory let you use a fifo for
 *    the source.
 */

/* On error handling and printing:
 *
 * The xdelta library sets stream->msg to indicate what condition caused an internal
 * failure, but many failures originate here and are printed here.  The return convention
 * is 0 for success, as throughout Xdelta code, but special attention is required here for
 * the operating system calls with different error handling.  See the main_file_* routines.
 * All errors in this file have a message printed at the time of occurance.  Since some of
 * these calls occur within calls to the library, the error may end up being printed again
 * with a more general error message.
 */

/******************************************************************************************/

#ifndef XD3_POSIX
#define XD3_POSIX 0
#endif
#ifndef XD3_STDIO
#define XD3_STDIO 0
#endif
#ifndef XD3_WIN32
#define XD3_WIN32 0
#endif

/* Combines xd3_strerror() and strerror() */
const char* xd3_mainerror(int err_num);

/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */
#define XPR fprintf
#define NT stderr, "xdelta3: "

/* If none are set, default to posix. */
#if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0
#undef XD3_POSIX
#define XD3_POSIX 1
#endif

/* Handle externally-compressed inputs. */
#ifndef EXTERNAL_COMPRESSION
#define EXTERNAL_COMPRESSION 1
#endif

#define PRINTHDR_SPECIAL -4378291

/* The number of soft-config variables.  */
#define XD3_SOFTCFG_VARCNT 7

/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an error message
 * from the library. */
#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", xd3_errstring (stream), xd3_mainerror (ret)

#include <stdio.h>  /* fprintf */

#if XD3_POSIX
#include <unistd.h> /* close, read, write... */
#include <sys/types.h>
#include <fcntl.h>
#endif

#ifndef _WIN32
#include <unistd.h> /* lots */
#include <sys/time.h> /* gettimeofday() */
#include <sys/stat.h> /* stat() and fstat() */
#else
#define strtoll _strtoi64
#include <sys/types.h>
#include <sys/stat.h>
#ifndef WIFEXITED
#   define WIFEXITED(stat)  (((*((int *) &(stat))) & 0xff) == 0)
#endif
#ifndef WEXITSTATUS
#   define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff)
#endif
#ifndef S_ISREG
//#   ifdef S_IFREG
//#       define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
//#   else
#       define S_ISREG(m) 1
//#   endif
#endif /* !S_ISREG */

// For standard input/output handles
static STARTUPINFO winStartupInfo;
#endif

/******************************************************************************************
 ENUMS and TYPES
 ******************************************************************************************/

/* These flags (mainly pertaining to main_read() operations) are set in the
 * main_file->flags variable.  All are related to with external decompression support.
 *
 * RD_FIRST causes the external decompression check when the input is first read.
 *
 * RD_NONEXTERNAL disables external decompression for reading a compressed input, in the
 * case of Xdelta inputs.  Note: Xdelta is supported as an external compression type,
 * which makes is the reason for this flag.  An example to justify this is: to create a
 * delta between two files that are VCDIFF-compressed.  Two external Xdelta decoders are
 * run to supply decompressed source and target inputs to the Xdelta encoder. */
typedef enum
{
  RD_FIRST        = (1 << 0),
  RD_NONEXTERNAL  = (1 << 1),
  RD_EXTERNAL_V1  = (1 << 2),
} xd3_read_flags;

/* main_file->mode values */
typedef enum
{
  XO_READ  = 0,
  XO_WRITE = 1,
} main_file_modes;

/* Main commands.  For example, CMD_PRINTHDR is the "xdelta printhdr" command. */
typedef enum
{
  CMD_NONE = 0,
  CMD_PRINTHDR,
  CMD_PRINTHDRS,
  CMD_PRINTDELTA,
  CMD_RECODE,
#if XD3_ENCODER
  CMD_ENCODE,
#endif
  CMD_DECODE,
  CMD_TEST,
  CMD_CONFIG,
} xd3_cmd;

#if XD3_ENCODER
#define CMD_DEFAULT CMD_ENCODE
#define IS_ENCODE(cmd) (cmd == CMD_ENCODE)
#else
#define CMD_DEFAULT CMD_DECODE
#define IS_ENCODE(cmd) (0)
#endif

typedef struct _main_file        main_file;
typedef struct _main_extcomp     main_extcomp;
typedef struct _main_blklru      main_blklru;
typedef struct _main_blklru_list main_blklru_list;

/* The main_file object supports abstract system calls like open, close, read, write, seek,
 * stat.  The program uses these to represent both seekable files and non-seekable files.
 * Source files must be seekable, but the target input and any output file do not require
 * seekability.
 */
struct _main_file
{
#if XD3_STDIO
  FILE               *file;
#elif XD3_POSIX
  int                 file;
#elif XD3_WIN32
  HANDLE              file;
#endif

  int                 mode;          /* XO_READ and XO_WRITE */
  const char         *filename;      /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
  char               *filename_copy; /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
  const char         *realname;      /* File name or /dev/stdin, /dev/stdout, /dev/stderr. */
  const main_extcomp *compressor;    /* External compression struct. */
  int                 flags;         /* RD_FIRST, RD_NONEXTERNAL, ... */
  xoff_t              nread;         /* for input position */
  xoff_t              nwrite;        /* for output position */
  uint8_t            *snprintf_buf;  /* internal snprintf() use */
};

/* Various strings and magic values used to detect and call external compression.  See
 * below for examples. */
struct _main_extcomp
{
  const char    *recomp_cmdname;
  const char    *recomp_options;

  const char    *decomp_cmdname;
  const char    *decomp_options;

  const char    *ident;
  const char    *magic;
  int            magic_size;
  int            flags;
};

/* This file implements a small LRU of source blocks.  For encoding purposes,
 * we prevent paging in blocks we've already scanned in the source (return
 * XD3_NOTAVAIL). */
struct _main_blklru_list
{
  main_blklru_list  *next;
  main_blklru_list  *prev;
};

struct _main_blklru
{
  uint8_t         *blk;
  xoff_t           blkno;
  main_blklru_list  link;
};

#define LRU_SIZE 32U
#define XD3_MINSRCWINSZ XD3_ALLOCSIZE

/* ... represented as a list (no cache index). */
XD3_MAKELIST(main_blklru_list,main_blklru,link);

// TODO:
// struct _main_state
// {

/* Program options: various command line flags and options. */
static int         option_stdout             = 0;
static int         option_force              = 0;
static int         option_verbose            = 0;
static int         option_quiet              = 0;
static int         option_use_appheader      = 1;
static uint8_t*    option_appheader          = NULL;
static int         option_use_secondary      = 0;
static char*       option_secondary          = NULL;
static int         option_use_checksum       = 1;
static int         option_use_altcodetable   = 0;
static char*       option_smatch_config      = NULL;
static int         option_no_compress        = 0;
static int         option_no_output          = 0; /* do not open or write output */
static const char *option_source_filename    = NULL;

static int         option_level              = XD3_DEFAULT_LEVEL;
static usize_t     option_iopt_size          = XD3_DEFAULT_IOPT_SIZE;
static usize_t     option_winsize            = XD3_DEFAULT_WINSIZE;
static usize_t     option_srcwinsz           = XD3_DEFAULT_SRCWINSZ;
static usize_t     option_sprevsz            = XD3_DEFAULT_SPREVSZ;

/* These variables are supressed to avoid their use w/o support.  main() warns
 * appropriately. */
#if EXTERNAL_COMPRESSION
static int         option_decompress_inputs  = 1;
static int         option_recompress_outputs = 1;
#endif

/* This is for comparing "printdelta" output without attention to
 * copy-instruction modes. */
#if VCDIFF_TOOLS
static int         option_print_cpymode = 1;
#endif

/* Static variables */
IF_DEBUG(static int main_mallocs = 0;)

static char*          program_name = NULL;
static uint8_t*       appheader_used = NULL;
static uint8_t*       main_bdata = NULL;

/* The LRU: obviously this is shared by all callers. */
static int               lru_size = 0;
static main_blklru      *lru = NULL;  /* array of lru_size elts */
static main_blklru_list  lru_list;
static main_blklru_list  lru_free;
static int               do_not_lru = 0;  /* set to avoid lru, instead discard oldest */

static int lru_hits   = 0;
static int lru_misses = 0;
static int lru_filled = 0;

/* Hacks for VCDIFF tools */
static int allow_fake_source = 0;

/* State for xdelta3 recode */
static xd3_stream *recode_stream = NULL;

/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is false just so
 * the program knows the mapping of IDENT->NAME. */
static main_extcomp extcomp_types[] =
{
  /* The entry for xdelta3 must be 0 because the program_name is set there. */
  { "xdelta3",  "-cfq",  "xdelta3",    "-dcfq",  "X", "\xd6\xc3\xc4", 3, RD_NONEXTERNAL },
  { "bzip2",    "-cf",   "bzip2",      "-dcf",   "B", "BZh",          3, 0 },
  { "gzip",     "-cf",   "gzip",       "-dcf",   "G", "\037\213",     2, 0 },
  { "compress", "-cf",   "uncompress", "-cf",    "Z", "\037\235",     2, 0 },

  /* TODO: add commandline support for magic-less formats */
  /*{ "lzma",     "-cf",   "lzma",       "-dcf",   "M", "]\000",        2, 0 },*/
};

// };

static void main_get_appheader (xd3_stream *stream, main_file *ifile,
                                main_file *output, main_file *sfile);

static int main_help (void);

static int
main_version (void)
{
  /* $Format: "  DP(RINT \"VERSION=3.$Xdelta3Version$\\n\");" $ */
  DP(RINT "VERSION=3.0r\n");
  return EXIT_SUCCESS;
}

static int
main_config (void)
{
  main_version ();

  DP(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);
  DP(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES);
  DP(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE);
  DP(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST);
  DP(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW);
  DP(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK);
  DP(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);
  DP(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
  DP(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);
  DP(RINT "XD3_ENCODER=%d\n", XD3_ENCODER);
  DP(RINT "XD3_POSIX=%d\n", XD3_POSIX);
  DP(RINT "XD3_STDIO=%d\n", XD3_STDIO);
  DP(RINT "XD3_WIN32=%d\n", XD3_WIN32);
  DP(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);
  DP(RINT "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL);
  DP(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
  DP(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
  DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
  DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
  DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);

  return EXIT_SUCCESS;
}

static void
reset_defaults(void)
{
  option_stdout = 0;
  option_force = 0;
  option_verbose = 0;
  option_quiet = 0;
  option_appheader = NULL;
  option_use_secondary = 0;
  option_secondary = NULL;
  option_use_altcodetable = 0;
  option_smatch_config = NULL;
  option_no_compress = 0;
  option_no_output = 0;
  option_source_filename = NULL;
  program_name = NULL;
  appheader_used = NULL;
  main_bdata = NULL;
  lru_size = 0;
  lru = NULL;
  do_not_lru = 0;
  lru_hits   = 0;
  lru_misses = 0;
  lru_filled = 0;
  allow_fake_source = 0;
  option_smatch_config = NULL;

  option_use_appheader = 1;
  option_use_checksum = 1;
#if EXTERNAL_COMPRESSION
  option_decompress_inputs  = 1;
  option_recompress_outputs = 1;
#endif
#if VCDIFF_TOOLS
  option_print_cpymode = 1;
#endif
  option_level = XD3_DEFAULT_LEVEL;
  option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
  option_winsize = XD3_DEFAULT_WINSIZE;
  option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
  option_sprevsz = XD3_DEFAULT_SPREVSZ;
}

static void*
main_malloc1 (usize_t size)
{
  void* r = malloc (size);
  if (r == NULL) { XPR(NT "malloc: %s\n", xd3_mainerror (ENOMEM)); }
  else if (option_verbose > 3) { XPR(NT "malloc: %u: %p\n", size, r); }
  return r;
}

static void*
main_malloc (usize_t size)
{
  void *r = main_malloc1 (size);
   if (r) { IF_DEBUG (main_mallocs += 1); }
  return r;
}

static void*
main_alloc (void   *opaque,
            usize_t  items,
            usize_t  size)
{
  return main_malloc1 (items * size);
}

static void
main_free1 (void *opaque, void *ptr)
{
  if (option_verbose > 3) { XPR(NT "free: %p\n", ptr); }
  free (ptr);
}

static void
main_free (void *ptr)
{
  if (ptr)
    {
      IF_DEBUG (main_mallocs -= 1);
      main_free1 (NULL, ptr);
      IF_DEBUG (XD3_ASSERT(main_mallocs >= 0));
    }
}

/* This ensures that (ret = errno) always indicates failure, in case errno was
 * accidentally not set.  If this prints there's a bug somewhere. */
static int
get_errno (void)
{
#ifndef _WIN32
  if (errno == 0)
    {
      XPR(NT "you found a bug: expected errno != 0\n");
      errno = XD3_INTERNAL;
    }
  return errno;
#else
  DWORD errNum = GetLastError();
  if (errNum == NO_ERROR) {
          errNum = XD3_INTERNAL;
  }
  return errNum;
#endif
}

const char*
xd3_mainerror(int err_num) {
#ifndef _WIN32
        const char* x = xd3_strerror (err_num);
        if (x != NULL) {
                return x;
        }
        return strerror(err_num);
#else
        static char err_buf[256];
        const char* x = xd3_strerror (err_num);
        if (x != NULL) {
                return x;
        }
        memset (err_buf, 0, 256);
        FormatMessage (FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
                NULL, err_num,
                MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT),
                err_buf, 256, NULL);
        return err_buf;
#endif
}

static long
get_millisecs_now (void)
{
#ifndef _WIN32
  struct timeval tv;

  gettimeofday (& tv, NULL);

  return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000;
#else
  // Found this in an example on www.codeproject.com
  // It doesn't matter that the offset is Jan 1, 1601
  // Result is the numbre of 100 nanosecond units
  // 100ns * 10,000 = 1ms
  SYSTEMTIME st;
  FILETIME ft;
  __int64 *pi = (__int64*)&ft;
  GetLocalTime(&st);
  SystemTimeToFileTime(&st, &ft);
  return (long)((*pi) / 10000);
#endif
}

/* Always >= 1 millisec, right? */
static long
get_millisecs_since (void)
{
  static long last = 0;
  long now = get_millisecs_now();
  long diff = now - last;
  last = now;
  return diff;
}

static char*
main_format_bcnt (xoff_t r, char *buf)
{
  static const char* fmts[] = { "B", "KB", "MB", "GB" };
  int i;

  for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1)
    {
      if (r <= (10 * 1024) || i == (-1 + (int)SIZEOF_ARRAY(fmts)))
            {
              sprintf (buf, "%"Q"u %s", r, fmts[i]);
              break;
            }
      r /= 1024;
    }
  return buf;
}

static char*
main_format_rate (xoff_t bytes, long millis, char *buf)
{
  xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0));
  static char lbuf[32];

  main_format_bcnt (r, lbuf);
  sprintf (buf, "%s/sec", lbuf);
  return buf;
}

static char*
main_format_millis (long millis, char *buf)
{
  if (millis < 1000)       { sprintf (buf, "%lu ms", millis); }
  else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); }
  else                     { sprintf (buf, "%lu sec", millis / 1000L); }
  return buf;
}

/* A safe version of strtol for xoff_t. */
static int
main_strtoxoff (const char* s, xoff_t *xo, char which)
{
  char *e;
  xoff_t x;

  XD3_ASSERT(s && *s != 0);

  {
    /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */
#if SIZEOF_XOFF_T == 4
    long xx = strtol (s, &e, 0);
#else
    long long xx = strtoll (s, &e, 0);
#endif

    if (xx < 0)
      {
        XPR(NT "-%c: negative integer: %s\n", which, s);
        return EXIT_FAILURE;
      }

    x = xx;
  }

  if (*e != 0)
    {
      XPR(NT "-%c: invalid integer: %s\n", which, s);
      return EXIT_FAILURE;
    }

  (*xo) = x;
  return 0;
}

static int
main_atou (const char* arg, usize_t *xo, usize_t low, usize_t high, char which)
{
  xoff_t x;
  int ret;

  if ((ret = main_strtoxoff (arg, & x, which))) { return ret; }

  if (x < low)
    {
      XPR(NT "-%c: minimum value: %u\n", which, low);
      return EXIT_FAILURE;
    }
  if (high == 0)
    {
      high = USIZE_T_MAX;
    }
  if (x > high)
    {
      XPR(NT "-%c: maximum value: %u\n", which, high);
      return EXIT_FAILURE;
    }
  (*xo) = (usize_t)x;
  return 0;
}

/******************************************************************************************
 FILE BASICS
 ******************************************************************************************/