Merge branch 'master' of https://github.com/google/oss-fuzz

4 files changed, 473 insertions, 0 deletions

diff --git a/projects/ffmpeg/Dockerfile b/projects/ffmpeg/Dockerfile
new file mode 100644
index 00000000..6543bc8e
--- /dev/null
+++ b/projects/ffmpeg/Dockerfile
@@ -0,0 +1,43 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM ossfuzz/base-libfuzzer
+MAINTAINER mmoroz@chromium.org
+RUN apt-get install -y make autoconf automake libtool build-essential \
+    libass-dev libfreetype6-dev libsdl1.2-dev \
+    libvdpau-dev libxcb1-dev libxcb-shm0-dev \
+    pkg-config texinfo libbz2-dev zlib1g-dev nasm yasm cmake mercurial wget \
+    xutils-dev libpciaccess-dev
+
+RUN git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg
+
+RUN wget ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.1.0.tar.bz2
+RUN git clone git://anongit.freedesktop.org/mesa/drm
+RUN git clone https://github.com/mstorsjo/fdk-aac.git
+RUN wget https://sourceforge.net/projects/lame/files/latest/download -O lame.tar.gz
+RUN git clone git://anongit.freedesktop.org/xorg/lib/libXext
+RUN git clone git://anongit.freedesktop.org/git/xorg/lib/libXfixes
+RUN git clone git://anongit.freedesktop.org/git/libva
+RUN git clone git://people.freedesktop.org/~aplattner/libvdpau
+RUN git clone https://chromium.googlesource.com/webm/libvpx
+RUN svn co http://svn.xiph.org/trunk/ogg
+RUN git clone git://git.xiph.org/opus.git
+RUN git clone git://git.xiph.org/theora.git
+RUN git clone git://git.xiph.org/vorbis.git
+RUN git clone git://git.videolan.org/git/x264.git
+RUN hg clone https://bitbucket.org/multicoreware/x265
+
+COPY build.sh group_seed_corpus.py $SRC/
diff --git a/projects/ffmpeg/build.sh b/projects/ffmpeg/build.sh
new file mode 100755
index 00000000..87e589dc
--- /dev/null
+++ b/projects/ffmpeg/build.sh
@@ -0,0 +1,291 @@
+#!/bin/bash -eux
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Build dependencies.
+export FFMPEG_DEPS_PATH=$SRC/ffmpeg_deps
+mkdir -p $FFMPEG_DEPS_PATH
+
+cd $SRC
+bzip2 -f -d alsa-lib-*
+tar xf alsa-lib-*
+cd alsa-lib-*
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared
+make clean
+make -j$(nproc) all
+make install
+
+cd $SRC/drm
+# Requires xutils-dev libpciaccess-dev
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/fdk-aac
+autoreconf -fiv
+./configure --prefix="$FFMPEG_DEPS_PATH" --disable-shared
+make clean
+make -j$(nproc) all
+make install
+
+cd $SRC
+tar xzf lame.tar.gz
+cd lame-*
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/libXext
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/libXfixes
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/libva
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared
+make clean
+make -j$(nproc) all
+make install
+
+cd $SRC/libvdpau
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-shared
+make clean
+make -j$(nproc) all
+make install
+
+cd $SRC/libvpx
+LDFLAGS="$CXXFLAGS $LDFLAGS" ./configure --prefix="$FFMPEG_DEPS_PATH" \
+    --disable-examples --disable-unit-tests
+make clean
+make -j$(nproc) all
+make install
+
+cd $SRC/ogg
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/opus
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static
+make clean
+make -j$(nproc) all
+make install
+
+cd $SRC/theora
+# theora requires ogg, need to pass its location to the "configure" script.
+CFLAGS="$CFLAGS -fPIC" LDFLAGS="$LDFLAGS -L$FFMPEG_DEPS_PATH/lib/" \
+    CPPFLAGS="$CXXFLAGS -I$FFMPEG_DEPS_PATH/include/" \
+    LD_LIBRARY_PATH="$FFMPEG_DEPS_PATH/lib/" \
+    ./autogen.sh --prefix="$FFMPEG_DEPS_PATH" --enable-static --disable-examples
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/vorbis
+./autogen.sh
+./configure --prefix="$FFMPEG_DEPS_PATH" --enable-static
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/x264
+LDFLAGS="$CXXFLAGS $LDFLAGS" ./configure --prefix="$FFMPEG_DEPS_PATH" \
+    --enable-static
+make clean
+make -j$(nproc)
+make install
+
+cd $SRC/x265/build/linux
+cmake -G "Unix Makefiles" \
+    -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX \
+    -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
+    -DCMAKE_INSTALL_PREFIX="$FFMPEG_DEPS_PATH" -DENABLE_SHARED:bool=off \
+    ../../source
+make clean
+make -j$(nproc) x265-static
+make install
+
+# Remove shared libraries to avoid accidental linking against them.
+rm $FFMPEG_DEPS_PATH/lib/*.so
+rm $FFMPEG_DEPS_PATH/lib/*.so.*
+
+# Build the target.
+cd $SRC/ffmpeg
+PKG_CONFIG_PATH="$FFMPEG_DEPS_PATH/lib/pkgconfig" ./configure \
+    --cc=$CC --cxx=$CXX --ld="$CXX $CXXFLAGS -std=c++11" \
+    --extra-cflags="-I$FFMPEG_DEPS_PATH/include" \
+    --extra-ldflags="-L$FFMPEG_DEPS_PATH/lib" \
+    --prefix="$FFMPEG_DEPS_PATH" \
+    --pkg-config-flags="--static" \
+    --enable-gpl \
+    --enable-libass \
+    --enable-libfdk-aac \
+    --enable-libfreetype \
+    --enable-libmp3lame \
+    --enable-libopus \
+    --enable-libtheora \
+    --enable-libvorbis \
+    --enable-libvpx \
+    --enable-libx264 \
+    --enable-libx265 \
+    --enable-nonfree \
+    --disable-shared
+make clean
+make -j$(nproc) install
+
+# Download test sampes, will be used as seed corpus.
+export TEST_SAMPLES_PATH=$SRC/ffmpeg/fate-suite/
+make fate-rsync SAMPLES=$TEST_SAMPLES_PATH
+
+# Build the fuzzers.
+cd $SRC/ffmpeg
+
+export TEMP_VAR_CODEC="AV_CODEC_ID_H264"
+export TEMP_VAR_CODEC_TYPE="VIDEO"
+
+FFMPEG_FUZZERS_COMMON_FLAGS="-lfuzzer /usr/local/lib/libc++.a \
+    -L$FFMPEG_DEPS_PATH/lib \
+    -Llibavcodec -Llibavdevice -Llibavfilter -Llibavformat -Llibavresample \
+    -Llibavutil -Llibpostproc -Llibswscale -Llibswresample \
+    -Wl,--as-needed -Wl,-z,noexecstack -Wl,--warn-common \
+    -Wl,-rpath-link=libpostproc:libswresample:libswscale:libavfilter:libavdevice:libavformat:libavcodec:libavutil:libavresample \
+    -lavdevice -lavfilter -lavformat -lavcodec -lswresample -lswscale \
+    -lavutil -ldl -lxcb -lxcb-shm -lxcb -lxcb-xfixes  -lxcb -lxcb-shape -lxcb \
+    -lX11 -lasound -lm -lbz2 -lz -pthread -lva-x11 -lXext -lXfixes \
+    -lx264 -lx265 -lvpx -lva -lvorbis -logg -lvorbisenc -lopus -lmp3lame \
+    -lfdk-aac -ltheora -ltheoraenc -ltheoradec -lvdpau -lva-drm -ldrm"
+
+# Build fuzzers for audio formats.
+CODEC_TYPE="AUDIO"
+CODEC_NAMES="AV_CODEC_ID_AAC \
+  AV_CODEC_ID_AC3 \
+  AV_CODEC_ID_ADPCM_ADX \
+  AV_CODEC_ID_AMR_NB \
+  AV_CODEC_ID_AMR_WB \
+  AV_CODEC_ID_DTS \
+  AV_CODEC_ID_EAC3 \
+  AV_CODEC_ID_FLAC \
+  AV_CODEC_ID_GSM_MS \
+  AV_CODEC_ID_MP2 \
+  AV_CODEC_ID_MP3 \
+  AV_CODEC_ID_QCELP \
+  AV_CODEC_ID_SIPR \
+  AV_CODEC_ID_WAVPACK"
+
+for codec in $CODEC_NAMES; do
+  fuzzer_name=ffmpeg_${CODEC_TYPE}_${codec}_fuzzer
+
+  $CC $CFLAGS -I${FFMPEG_DEPS_PATH}/include \
+      $SRC/ffmpeg/doc/examples/decoder_targeted.c \
+      -o $OUT/${fuzzer_name} \
+      -DFFMPEG_CODEC=${codec} -DFUZZ_FFMPEG_${CODEC_TYPE}= \
+      ${FFMPEG_FUZZERS_COMMON_FLAGS}
+
+  echo -en "[libfuzzer]\nmax_len = 1000000\n" > $OUT/${fuzzer_name}.options
+done
+
+# Build fuzzers for subtitles formats.
+CODEC_TYPE="SUBTITLE"
+CODEC_NAMES="AV_CODEC_ID_DVD_SUBTITLE \
+  AV_CODEC_ID_MOV_TEXT \
+  AV_CODEC_ID_SUBRIP"
+
+for codec in $CODEC_NAMES; do
+  fuzzer_name=ffmpeg_${CODEC_TYPE}_${codec}_fuzzer
+
+  $CC $CFLAGS -I${FFMPEG_DEPS_PATH}/include \
+      $SRC/ffmpeg/doc/examples/decoder_targeted.c \
+      -o $OUT/${fuzzer_name} \
+      -DFFMPEG_CODEC=${codec} -DFUZZ_FFMPEG_${CODEC_TYPE}= \
+      ${FFMPEG_FUZZERS_COMMON_FLAGS}
+done
+
+# Build fuzzers for video formats.
+CODEC_TYPE="VIDEO"
+CODEC_NAMES="AV_CODEC_ID_AMV \
+  AV_CODEC_ID_BINTEXT \
+  AV_CODEC_ID_BMP \
+  AV_CODEC_ID_CINEPAK \
+  AV_CODEC_ID_DVVIDEO \
+  AV_CODEC_ID_ESCAPE130 \
+  AV_CODEC_ID_FLIC \
+  AV_CODEC_ID_FLV1 \
+  AV_CODEC_ID_FRAPS \
+  AV_CODEC_ID_GIF \
+  AV_CODEC_ID_H263 \
+  AV_CODEC_ID_H263I \
+  AV_CODEC_ID_H264 \
+  AV_CODEC_ID_INDEO2 \
+  AV_CODEC_ID_INTERPLAY_VIDEO \
+  AV_CODEC_ID_JPEGLS \
+  AV_CODEC_ID_KMVC \
+  AV_CODEC_ID_MDEC \
+  AV_CODEC_ID_MJPEG \
+  AV_CODEC_ID_MPEG1VIDEO \
+  AV_CODEC_ID_MPEG2VIDEO \
+  AV_CODEC_ID_MPEG4 \
+  AV_CODEC_ID_MSVIDEO1 \
+  AV_CODEC_ID_PCX \
+  AV_CODEC_ID_PGM \
+  AV_CODEC_ID_PICTOR \
+  AV_CODEC_ID_PNG \
+  AV_CODEC_ID_RPZA \
+  AV_CODEC_ID_RV40 \
+  AV_CODEC_ID_SANM \
+  AV_CODEC_ID_SMC \
+  AV_CODEC_ID_SUNRAST \
+  AV_CODEC_ID_SVQ1 \
+  AV_CODEC_ID_SVQ3 \
+  AV_CODEC_ID_TARGA \
+  AV_CODEC_ID_TIFF \
+  AV_CODEC_ID_VP3 \
+  AV_CODEC_ID_VP5 \
+  AV_CODEC_ID_VP6 \
+  AV_CODEC_ID_VP6F \
+  AV_CODEC_ID_VP8 \
+  AV_CODEC_ID_ZMBV"
+
+for codec in $CODEC_NAMES; do
+  fuzzer_name=ffmpeg_${CODEC_TYPE}_${codec}_fuzzer
+
+  $CC $CFLAGS -I${FFMPEG_DEPS_PATH}/include \
+      $SRC/ffmpeg/doc/examples/decoder_targeted.c \
+      -o $OUT/${fuzzer_name} \
+      -DFFMPEG_CODEC=${codec} -DFUZZ_FFMPEG_${CODEC_TYPE}= \
+      ${FFMPEG_FUZZERS_COMMON_FLAGS}
+
+  echo -en "[libfuzzer]\nmax_len = 1000000\n" > $OUT/${fuzzer_name}.options
+done
+
+# Find relevant corpus in test samples and archive them for every fuzzer.
+cd $SRC
+python group_seed_corpus.py $TEST_SAMPLES_PATH $OUT/
diff --git a/projects/ffmpeg/group_seed_corpus.py b/projects/ffmpeg/group_seed_corpus.py
new file mode 100755
index 00000000..1e1d51cd
--- /dev/null
+++ b/projects/ffmpeg/group_seed_corpus.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+from __future__ import print_function
+import logging
+import os
+import re
+import sys
+import zipfile
+
+
+logging.basicConfig(level=logging.INFO, format='INFO: %(message)s')
+CODEC_NAME_REGEXP = re.compile(r'codec_id_(.+?)_fuzzer')
+
+
+def get_fuzzer_tags(fuzzer_name):
+  """Extract tags (are used to filter samples) from the given fuzzer name."""
+  tags = []
+  fuzzer_name = fuzzer_name.lower()
+  # All subtitle samples are in 'sub' directory, need to add 'sub' tag manually.
+  if 'subtitle' in fuzzer_name:
+    tags.append('sub')
+  m = CODEC_NAME_REGEXP.search(fuzzer_name)
+  if m:
+    codec_name = m.group(1)
+    # Some names are complex, need to split them and filter common strings.
+    codec_name_parts = codec_name.split('_')
+    for codec in codec_name_parts:
+      # Remove common strings from codec names like 'mpeg1video' or 'msvideo1'.
+      codec = codec.split('video')[0]
+      codec = codec.split('audio')[0]
+      codec = codec.split('subtitle')[0]
+      codec = codec.split('text')[0]
+      if codec:
+        # Some codec names have trailing characters: 'VP6F','FLV1', 'JPEGLS'.
+        # Use only first 3 characters for long enough codec names.
+        if len(codec) > 3:
+          tags.append(codec[:3])
+        else:
+          tags.append(codec)
+
+  return tags
+
+
+def parse_corpus(corpus_directory):
+  """Recursively list all files in the given directory and ignore checksums."""
+  all_corpus_files = []
+  for root, dirs, files in os.walk(corpus_directory):
+    for filename in files:
+      # Skip checksum files, they are useless in corpus.
+      if 'md5sum' in filename:
+        continue
+      path = os.path.join(root, filename)
+      all_corpus_files.append(path)
+
+  logging.info('Parsed %d corpus files from %s' % (len(all_corpus_files),
+                                                   corpus_directory))
+  return all_corpus_files
+
+
+def parse_fuzzers(fuzzers_directory):
+  """Recursively list all fuzzers in the given directory."""
+  all_fuzzers = []
+  for filename in os.listdir(fuzzers_directory):
+    # Skip non-ffmpeg and non-fuzzer files in the given directory,
+    if not filename.startswith('ffmpeg_') or not filename.endswith('_fuzzer'):
+      continue
+    fuzzer_path = os.path.join(fuzzers_directory, filename)
+    all_fuzzers.append(fuzzer_path)
+
+  logging.info('Parsed %d fuzzers from %s' % (len(all_fuzzers),
+                                              fuzzers_directory))
+  return all_fuzzers
+ 
+
+def zip_relevant_corpus(corpus_files, fuzzers):
+  """Find relevant corpus files and archive them for every fuzzer given."""
+  for fuzzer in fuzzers:
+    fuzzer_name = os.path.basename(fuzzer)
+    fuzzer_directory = os.path.dirname(fuzzer)
+    fuzzer_tags = get_fuzzer_tags(fuzzer_name)
+    relevant_corpus_files = set()
+    for filename in corpus_files:
+      # Remove 'ffmpeg' substring to do not use everything for 'MPEG' codec.
+      sanitized_filename = filename.replace('ffmpeg', '').lower()
+      for tag in fuzzer_tags:
+        if tag in sanitized_filename:
+          relevant_corpus_files.add(filename)
+
+      if not relevant_corpus_files:
+        # Strip last symbol from tags if we haven't found relevant corpus.
+        # It helps for such codecs as 'RV40' ('RV4' -> 'RV') or 'PCX' (-> 'PC').
+        for tag in fuzzer_tags:
+          if tag[:-1] in sanitized_filename:
+            relevant_corpus_files.add(filename)
+
+    logging.info(
+        'Found %d relevant samples for %s' % (len(relevant_corpus_files),
+                                              fuzzer_name))
+
+    if not relevant_corpus_files:
+      continue
+
+    zip_archive_name = fuzzer + "_seed_corpus.zip"
+    with zipfile.ZipFile(zip_archive_name, 'w') as archive:
+      for filename in relevant_corpus_files:
+        archive.write(filename)
+
+
+def main():
+  if len(sys.argv) < 3:
+    print('Usage: %s <seed_corpus_directory> <fuzzers_directory>' % __file__)
+    sys.exit(1)
+
+  seed_corpus_directory = sys.argv[1]
+  fuzzers_directory = sys.argv[2]
+
+  corpus_files = parse_corpus(seed_corpus_directory)
+  fuzzers = parse_fuzzers(fuzzers_directory)
+  zip_relevant_corpus(corpus_files, fuzzers)
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/projects/ffmpeg/target.yaml b/projects/ffmpeg/target.yaml
new file mode 100644
index 00000000..1a0131c9
--- /dev/null
+++ b/projects/ffmpeg/target.yaml
@@ -0,0 +1 @@
+homepage: "https://www.ffmpeg.org/"