github/hashcat
1
Fork 0
mirror of https://github.com/hashcat/hashcat synced 2024-11-24 14:27:14 +01:00
Code Releases Activity

Merge pull request #2960 from jtojanen/chunked

Browse Source 
Refactor file reading/writing
This commit is contained in:
Jens Steube 2021-09-03 10:10:08 +02:00 committed by GitHub
commit 82b337fbca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 94 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
src/filehandling.c
View File

@ -25,6 +25,10 @@ _Static_assert(sizeof (size_t) == sizeof (SizeT), "Check why sizeof(size_t) != s
#define HCFILE_BUFFER_SIZE 256 * 1024
#endif
#ifndef HCFILE_CHUNK_SIZE
#define HCFILE_CHUNK_SIZE 4 * 1024 * 1024
#endif
static bool xz_initialized = false;
static const ISzAlloc xz_alloc = { hc_lzma_alloc, hc_lzma_free };
@ -352,51 +356,47 @@ bool hc_fopen_raw (HCFILE *fp, const char *path, const char *mode)
size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp)
{
size_t n = -1;
size_t n = (size_t) -1;
if (fp == NULL) return n;
if (ptr == NULL || fp == NULL) return n;
if (ptr == NULL || size == 0 || nmemb == 0) return 0;
if (size == 0 || nmemb == 0) return 0;
if (fp->pfp)
{
#if defined (_WIN)
#ifdef _WIN
u64 len = (u64) size * nmemb;
// 4 GB fread () limit for windows systems ?
// see: https://social.msdn.microsoft.com/Forums/vstudio/en-US/7c913001-227e-439b-bf07-54369ba07994/fwrite-issues-with-large-data-write
#ifndef _WIN64
/* check 2 GB limit with 32 bit build */
if (len >= INT32_MAX) return n;
#endif
#define GIGABYTE (1024u * 1024u * 1024u)
if (((size * nmemb) / GIGABYTE) < 4)
if (len <= HCFILE_CHUNK_SIZE)
{
n = fread (ptr, size, nmemb, fp->pfp);
}
else
{
if ((size / GIGABYTE) > 3) return -1;
size_t left = (size_t) len;
size_t pos = 0;
size_t elements_max = (3u * GIGABYTE) / size;
size_t elements_left = nmemb;
/* assume success */
n = nmemb;
size_t off = 0;
n = 0;
while (elements_left > 0)
do
{
size_t elements_cur = elements_max;
if (elements_left < elements_max) elements_cur = elements_left;
size_t ret = fread (ptr + off, size, elements_cur, fp->pfp);
if (ret != elements_cur) return -1;
n += ret;
off += ret * size;
elements_left -= ret;
}
size_t chunk = (left > HCFILE_CHUNK_SIZE) ? HCFILE_CHUNK_SIZE : left;
size_t bytes = fread ((unsigned char *) ptr + pos, 1, chunk, fp->pfp);
pos += bytes;
left -= bytes;
if (chunk != bytes)
{
/* partial read */
n = pos / size;
break;
}
} while (left);
}
#else
n = fread (ptr, size, nmemb, fp->pfp);
@ -408,9 +408,31 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp)
}
else if (fp->ufp)
{
unsigned s = size * nmemb;
u64 len = (u64) size * nmemb;
u64 pos = 0;
n = unzReadCurrentFile (fp->ufp, ptr, s);
#if defined(_WIN) && !defined(_WIN64)
/* check 2 GB limit with 32 bit build */
if (len >= INT32_MAX) return n;
#endif
/* assume success */
n = nmemb;
do
{
unsigned chunk = (len > INT_MAX) ? INT_MAX : (unsigned) len;
int result = unzReadCurrentFile (fp->ufp, (unsigned char *) ptr + pos, chunk);
if (result < 0) return (size_t) -1;
pos += (u64) result;
len -= (u64) result;
if (chunk != (unsigned) result)
{
/* partial read */
n = pos / size;
break;
}
} while (len);
}
else if (fp->xfp)
{
@ -420,6 +442,14 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp)
SRes res = SZ_OK;
xzfile_t *xfp = fp->xfp;
#if defined(_WIN) && !defined(_WIN64)
/* check 2 GB limit with 32 bit build */
if (outLen >= INT32_MAX) return n;
#endif
/* assume success */
n = nmemb;
do
{
/* fill buffer if needed */
@ -438,13 +468,16 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp)
res = XzUnpacker_Code (&xfp->state, outBuf + outPos, &outLeft, xfp->inBuf + xfp->inPos, &inLeft, inLeft == 0, CODER_FINISH_ANY, &status);
xfp->inPos += inLeft;
xfp->inProcessed += inLeft;
if (res != SZ_OK) return -1;
if (inLeft == 0 && outLeft == 0) break;
if (res != SZ_OK) return (size_t) -1;
if (inLeft == 0 && outLeft == 0)
{
/* partial read */
n = (size_t) (outPos / size);
break;
}
outPos += outLeft;
xfp->outProcessed += outLeft;
} while (outPos < outLen);
n = outPos;
}
return n;
@ -454,47 +487,43 @@ size_t hc_fwrite (const void *ptr, size_t size, size_t nmemb, HCFILE *fp)
{
size_t n = -1;
if (fp == NULL) return n;
if (ptr == NULL || fp == NULL) return n;
if (size == 0 || nmemb == 0) return 0;
if (fp->pfp)
{
#if defined (_WIN)
#ifdef _WIN
u64 len = (u64) size * nmemb;
// 4 GB fwrite () limit for windows systems ?
// see: https://social.msdn.microsoft.com/Forums/vstudio/en-US/7c913001-227e-439b-bf07-54369ba07994/fwrite-issues-with-large-data-write
#ifndef _WIN64
/* check 2 GB limit with 32 bit build */
if (len >= INT32_MAX)
{
return n;
}
#endif
#define GIGABYTE (1024u * 1024u * 1024u)
if (((size * nmemb) / GIGABYTE) < 4)
if (len <= HCFILE_CHUNK_SIZE)
{
n = fwrite (ptr, size, nmemb, fp->pfp);
}
else
{
if ((size / GIGABYTE) > 3) return -1;
size_t left = (size_t) len;
size_t pos = 0;
size_t elements_max = (3u * GIGABYTE) / size;
size_t elements_left = nmemb;
/* assume success */
n = nmemb;
size_t off = 0;
n = 0;
while (elements_left > 0)
do
{
size_t elements_cur = elements_max;
if (elements_left < elements_max) elements_cur = elements_left;
size_t ret = fwrite (ptr + off, size, elements_cur, fp->pfp);
if (ret != elements_cur) return -1;
n += ret;
off += ret * size;
elements_left -= ret;
}
size_t chunk = (left > HCFILE_CHUNK_SIZE) ? HCFILE_CHUNK_SIZE : left;
size_t bytes = fwrite ((unsigned char *) ptr + pos, 1, chunk, fp->pfp);
pos += bytes;
left -= bytes;
if (chunk != bytes) return -1;
} while (left);
}
#else
n = fwrite (ptr, size, nmemb, fp->pfp);
@ -623,7 +652,7 @@ int hc_fstat (HCFILE *fp, struct stat *buf)
{
/* check that the uncompressed size is known */
const xzfile_t *xfp = fp->xfp;
if (xfp->outSize != (UInt64)((Int64)-1))
if (xfp->outSize != (UInt64) ((Int64) -1))
{
buf->st_size = (off_t) xfp->outSize;
}