Debian Bug report logs - #740059
clamav: LibClamAV Error: CRITICAL: fmap() failed

Reply or subscribe to this bug.

Toggle useless messages

Message #5 received at submit@bugs.debian.org (full text, mbox, reply):

From: Sthu <sthu.deus@yandex.ru>

To: Debian Bug Tracking System <submit@bugs.debian.org>

Subject: clamav: LibClamAV Error: CRITICAL: fmap() failed

Date: Tue, 25 Feb 2014 17:08:56 +0700

Package: clamav
Version: 0.98.1+dfsg-1
Severity: normal

Dear Maintainer,

FYI: about three weeks (after update) i see this behavior:

/usr/bin/clamscan -r --bell --infected --detect-pua=yes --scan-elf=yes --scan-mail=yes --algorithmic-detection=yes --scan-pe=yes --scan-ole2=yes --scan-pdf=yes --scan-html=yes --scan-archive=yes ./dir

LibClamAV Warning: fmap: map allocation failed
LibClamAV Error: CRITICAL: fmap() failed
LibClamAV Warning: fmap: map allocation failed
LibClamAV Error: CRITICAL: fmap() failed

----------- SCAN SUMMARY -----------
Known viruses: 3137431
Engine version: 0.98.1
Scanned directories: 2
Scanned files: 1
Infected files: 0
Total errors: 2
Data scanned: 0.15 MB
Data read: 2736.85 MB (ratio 0.00:1)
Time: 12.648 sec (0 m 12 s)

Thank you for your work for Debian!


-- Package-specific info:
--- configuration ---
#Automatically Generated by clamav-base postinst
#To reconfigure clamd run #dpkg-reconfigure clamav-base
#Please read /usr/share/doc/clamav-base/README.Debian.gz for details
LocalSocket /var/run/clamav/clamd.ctl
FixStaleSocket true
LocalSocketGroup clamav
LocalSocketMode 666
# TemporaryDirectory is not set to its default /tmp here to make overriding
# the default with environment variables TMPDIR/TMP/TEMP possible
User clamav
AllowSupplementaryGroups true
ScanMail true
ScanArchive true
ArchiveBlockEncrypted false
MaxDirectoryRecursion 15
FollowDirectorySymlinks false
FollowFileSymlinks false
ReadTimeout 180
MaxThreads 12
MaxConnectionQueueLength 15
LogSyslog false
LogFacility LOG_LOCAL6
LogClean false
LogVerbose false
PidFile /var/run/clamav/clamd.pid
DatabaseDirectory /var/lib/clamav
SelfCheck 3600
Foreground false
Debug false
ScanPE true
ScanOLE2 true
ScanHTML true
DetectBrokenExecutables false
ExitOnOOM false
LeaveTemporaryFiles false
AlgorithmicDetection true
ScanELF true
IdleTimeout 30
PhishingSignatures true
PhishingScanURLs true
PhishingAlwaysBlockSSLMismatch false
PhishingAlwaysBlockCloak false
DetectPUA false
ScanPartialMessages false
HeuristicScanPrecedence false
StructuredDataDetection false
CommandReadTimeout 5
SendBufTimeout 200
MaxQueue 100
ExtendedDetectionInfo true
OLE2BlockMacros false
StreamMaxLength 25M
LogFile /var/log/clamav/clamav.log
LogTime true
LogFileUnlock false
LogFileMaxSize 0
Bytecode true
BytecodeSecurity TrustSigned
BytecodeTimeout 60000
OfficialDatabaseOnly false
CrossFilesystems true
# Automatically created by the clamav-freshclam postinst
# Comments will get lost when you reconfigure the clamav-freshclam package

DatabaseOwner clamav
UpdateLogFile /dev/null
LogVerbose false
LogSyslog false
LogFacility LOG_LOCAL6
LogFileMaxSize 0
LogTime true
Foreground false
Debug false
MaxAttempts 5
DatabaseDirectory /var/lib/clamav
DNSDatabaseInfo current.cvd.clamav.net
AllowSupplementaryGroups false
PidFile /var/run/clamav/freshclam.pid
ConnectTimeout 30
ReceiveTimeout 30
TestDatabases yes
ScriptedUpdates yes
CompressLocalDatabase no
Bytecode true
# Check for new database 24 times a day
Checks 24
DatabaseMirror db.local.clamav.net
DatabaseMirror database.clamav.net

--- data dir ---
total 204756
-rw-r--r--. 1 clamav clamav    345088 Feb  6 01:48 bytecode.cld
drwxr-xr-x. 3 clamav clamav      4096 Feb 21 23:11 clamav-248394ea381111a7d480d04f5805aa02.tmp
drwxr-xr-x. 2 clamav clamav      4096 Oct 11 21:48 clamav-f4711d8420bc5ae3a427e583debfcec5
-rw-r--r--. 1 clamav clamav  45827072 Feb 25 00:24 daily.cld
-rw-r--r--. 1 clamav clamav 163468288 Sep 18 14:58 main.cld
-rw-r--r--. 1 clamav clamav       936 Feb 25 16:32 mirrors.dat

-- System Information:
Debian Release: jessie/sid
  APT prefers testing-updates
  APT policy: (700, 'testing-updates'), (700, 'testing'), (600, 'stable-updates'), (600, 'stable')
Architecture: amd64 (x86_64)
Foreign Architectures: i386

Kernel: Linux 3.12-1-amd64 (SMP w/2 CPU cores)
Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash

Versions of packages clamav depends on:
ii  clamav-freshclam [clamav-data]  0.98.1+dfsg-1
ii  libc6                           2.17-97
ii  libclamav6                      0.98.1+dfsg-1
ii  zlib1g                          1:1.2.8.dfsg-1

Versions of packages clamav recommends:
ii  clamav-base  0.98.1+dfsg-1

Versions of packages clamav suggests:
pn  clamav-docs  <none>

-- no debconf information

Message #10 received at 740059@bugs.debian.org (full text, mbox, reply):

From: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>

To: Sthu <sthu.deus@yandex.ru>

Cc: 740059@bugs.debian.org

Subject: Re: clamav: LibClamAV Error: CRITICAL: fmap() failed

Date: Wed, 12 Mar 2014 20:19:38 +0100

On 2014-02-25 17:08:56 [+0700], Sthu wrote:
> FYI: about three weeks (after update) i see this behavior:
> 
> /usr/bin/clamscan -r --bell --infected --detect-pua=yes --scan-elf=yes --scan-mail=yes --algorithmic-detection=yes --scan-pe=yes --scan-ole2=yes --scan-pdf=yes --scan-html=yes --scan-archive=yes ./dir
> 
> LibClamAV Warning: fmap: map allocation failed
> LibClamAV Error: CRITICAL: fmap() failed
> LibClamAV Warning: fmap: map allocation failed
> LibClamAV Error: CRITICAL: fmap() failed
> 
> ----------- SCAN SUMMARY -----------
> Known viruses: 3137431
> Engine version: 0.98.1
> Scanned directories: 2
> Scanned files: 1
> Infected files: 0
> Total errors: 2
> Data scanned: 0.15 MB
> Data read: 2736.85 MB (ratio 0.00:1)
> Time: 12.648 sec (0 m 12 s)

I tried a few things and didn't manage to reproduce this. It seems you
try to scan one file which is approx. 2.7 GiB in size and the error
message you quoted comes if you ran out of memory. Could you please
state the amount of memory you have. Is it possible to share that file
you try scan? If not can you give something else that shows that error?

Sebastian

Message #15 received at 740059@bugs.debian.org (full text, mbox, reply):

From: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>

To: Ста Деюс <sthu.deus@yandex.ru>

Cc: 740059@bugs.debian.org

Subject: Re: clamav: LibClamAV Error: CRITICAL: fmap() failed

Date: Sat, 15 Mar 2014 20:37:01 +0100

On 2014-03-16 00:16:38 [+0700], Ста Деюс wrote:
> Здравствуй, Sebastian.

Hello Ста,

> You have guessed right: i less than 2.7 GB of RAM, while i was
> scanning file of 2.7 GB in size.

What kind of file is it? Is this some kind of raw data or an archive?

> So, you can assuredly close my report. Thank you for your work for
> Debian!

So you definitly do not want any more investigation on this? From my
point of view it should be possible to scan a file which larger than
your RAM.

> Хорошего времени суток.
> 
> 
> С уважением,
> Ста.

Sebastian

Message #20 received at 740059@bugs.debian.org (full text, mbox, reply):

From: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>

To: 740059@bugs.debian.org

Cc: Ста Деюс <sthu.deus@yandex.ru>

Subject: Re: Bug#740059: clamav: LibClamAV Error: CRITICAL: fmap() failed

Date: Fri, 11 Apr 2014 22:07:07 +0200

tags 740059 + upstream
forwarded 740059 https://bugzilla.clamav.net/show_bug.cgi?id=10971
thanks

On 2014-03-15 20:37:01 [+0100], Sebastian Andrzej Siewior wrote:
> On 2014-03-16 00:16:38 [+0700], Ста Деюс wrote:
> > Здравствуй, Sebastian.
 
Hello Ста,

the patch below is what I sent to clamav's bugzilla. It drops the memory
constrain a little which means it should be fine on 64bit. Now I wait what
upstream says.

From 0a6fe67b724919a8c5583a762b7e10330c06b271 Mon Sep 17 00:00:00 2001
From: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Date: Thu, 10 Apr 2014 23:08:02 +0200
Subject: [PATCH] libclamav: fmap: try to mmap() the file instead read it
 completly

On unix platform the fmap_check_empty() function maps the complete file
by allocating the memory via mmap() with the write flag on. On systems
with not a lot of memory it means that a file can't be scanned. For
instance a 1GiB file on a system with 512MiB of memory.
This patch changes the behaviour to what we have on win32:
- the malloc() cl_fmap_t structure
- mmap() the file as RO only. Since it content is backed by file on disk
  we mmap() any file as long as our virtual memory space is large
  enough. That means this will fail on 32bit architectures which try
  scan a file which is ~2.8GiB. This failed before as well but now we
  aleast not limited by our physical memory.

Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
---
 libclamav/fmap.c |   85 +++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 71 insertions(+), 14 deletions(-)

diff --git a/libclamav/fmap.c b/libclamav/fmap.c
index 09b2306..929b496 100644
--- a/libclamav/fmap.c
+++ b/libclamav/fmap.c
@@ -52,6 +52,7 @@ static inline unsigned int fmap_which_page(fmap_t *m, size_t at);
 /* pread proto here in order to avoid the use of XOPEN and BSD_SOURCE
    which may in turn prevent some mmap constants to be defined */
 ssize_t pread(int fd, void *buf, size_t count, off_t offset);
+static cl_fmap_t *cl_fmap_mmap_fd(int fd, size_t offset, size_t len);
 
 /* vvvvv POSIX STUFF BELOW vvvvv */
 static off_t pread_cb(void *handle, void *buf, size_t count, off_t offset)
@@ -61,12 +62,8 @@ static off_t pread_cb(void *handle, void *buf, size_t count, off_t offset)
 
 
 fmap_t *fmap_check_empty(int fd, off_t offset, size_t len, int *empty) {
-    unsigned int pages, mapsz, hdrsz;
-    unsigned short dumb = 1;
-    int pgsz = cli_getpagesize();
     STATBUF st;
     fmap_t *m;
-    void *handle = (void*)(ssize_t)fd;
 
     *empty = 0;
     if(FSTAT(fd, &st)) {
@@ -84,10 +81,11 @@ fmap_t *fmap_check_empty(int fd, off_t offset, size_t len, int *empty) {
 	cli_warnmsg("fmap: attempted oof mapping\n");
 	return NULL;
     }
-    m = cl_fmap_open_handle((void*)(ssize_t)fd, offset, len, pread_cb, 1);
+    m = cl_fmap_mmap_fd(fd, offset, len);
     if (!m)
 	return NULL;
     m->mtime = st.st_mtime;
+    m->handle = (void*)(long)fd;
     m->handle_is_fd = 1;
     return m;
 }
@@ -161,6 +159,15 @@ fmap_t *fmap_check_empty(int fd, off_t offset, size_t len, int *empty) { /* WIN3
 }
 #endif /* _WIN32 */
 
+/* vvvvv MEMORY STUFF BELOW vvvvv */
+
+static const void *mem_need(fmap_t *m, size_t at, size_t len, int lock);
+static void mem_unneed_off(fmap_t *m, size_t at, size_t len);
+static const void *mem_need_offstr(fmap_t *m, size_t at, size_t len_hint);
+static const void *mem_gets(fmap_t *m, char *dst, size_t *at, size_t max_len);
+
+static void unmap_none(fmap_t *m) {}
+
 /* vvvvv SHARED STUFF BELOW vvvvv */
 
 #define FM_MASK_COUNT 0x3fffffff
@@ -277,6 +284,65 @@ extern cl_fmap_t *cl_fmap_open_handle(void *handle, size_t offset, size_t len,
     return m;
 }
 
+static void unmap_free_mmap(fmap_t *m)
+{
+	void *data = (void *)m->data;
+	munmap(data, m->real_len);
+	free(m);
+}
+
+static cl_fmap_t *cl_fmap_mmap_fd(int fd, size_t offset, size_t len)
+{
+    cl_fmap_t *m;
+    void *data;
+    int pgsz = cli_getpagesize();
+
+    if(offset < 0 || offset != fmap_align_to(offset, pgsz)) {
+	cli_warnmsg("fmap: attempted mapping with unaligned offset\n");
+	return NULL;
+    }
+    if(!len) {
+	cli_dbgmsg("fmap: attempted void mapping\n");
+	return NULL;
+    }
+    if (offset >= len) {
+	cli_warnmsg("fmap: attempted oof mapping\n");
+	return NULL;
+    }
+
+    m = cli_calloc(1, sizeof(*m));
+    if (!m) {
+	    cli_warnmsg("fmap: map allocation failed 1\n");
+	    return NULL;
+    }
+
+    data = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, offset);
+    if (data == MAP_FAILED) {
+	    cli_warnmsg("fmap: mmap() in file failed: %m\n");
+	    goto err_free;
+    }
+
+    if(!m) {
+	cli_warnmsg("fmap: map allocation failed 2\n");
+	return NULL;
+    }
+    m->data = data;
+    m->len = len;
+    m->real_len = len;
+    m->pgsz = pgsz;
+    m->pages = fmap_align_items(len, pgsz);;
+    m->unmap = unmap_free_mmap;
+    m->need = mem_need;
+    m->need_offstr = mem_need_offstr;
+    m->gets = mem_gets;
+    m->unneed_off = mem_unneed_off;
+    m->offset = offset;
+    return m;
+err_free:
+    free(m);
+    return NULL;
+}
+
 static void fmap_aging(fmap_t *m) {
 #ifdef ANONYMOUS_MAP
     if(!m->aging) return;
@@ -639,15 +705,6 @@ static const void *handle_gets(fmap_t *m, char *dst, size_t *at, size_t max_len)
     return dst;
 }
 
-/* vvvvv MEMORY STUFF BELOW vvvvv */
-
-static const void *mem_need(fmap_t *m, size_t at, size_t len, int lock);
-static void mem_unneed_off(fmap_t *m, size_t at, size_t len);
-static const void *mem_need_offstr(fmap_t *m, size_t at, size_t len_hint);
-static const void *mem_gets(fmap_t *m, char *dst, size_t *at, size_t max_len);
-
-static void unmap_none(fmap_t *m) {}
-
 extern cl_fmap_t *cl_fmap_open_memory(const void *start, size_t len)
 {
     int pgsz = cli_getpagesize();
-- 
1.7.10.4

Send a report that this bug log contains spam.