github
/
metasploit-framework
mirror of https://github.com/rapid7/metasploit-framework
1
Fork 0
Code Releases Activity

Remove fastlib 

MSP-11368
MSP-11143

Remove fastlib as it slows down the code loading process.  From the
previous commit, the mean loading for
`METASPLOIT_FRAMEWORK_PROFILE=true msfconsole -q -x exit` was
27.9530±0.3485 seconds (N=10).  The mean after removal of fastlib
was 17.9820±0.6497 seconds (N=10).  This means an average 35.67%
reduction in boot time.
This commit is contained in:
Luke Imhoff 2014-09-18 14:46:20 -05:00
parent 21b3d6bb4a
commit b863978028
No known key found for this signature in database
GPG Key ID: 5B1FB01FB33356F8
60 changed files with 10 additions and 1178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.simplecov
View File

@ -39,7 +39,6 @@ SimpleCov.configure do
# Other library groups
#
add_group 'Fastlib', 'lib/fastlib'
add_group 'Metasm', 'lib/metasm'
add_group 'PacketFu', 'lib/packetfu'
add_group 'Rex', 'lib/rex'

4
LICENSE
View File

@ -87,10 +87,6 @@ Files: lib/bit-struct.rb lib/bit-struct/*
Copyright: 2005-2009, Joel VanderWerf
License: Ruby
Files: lib/fastlib.rb
Copyright: 2011, Rapid7, Inc.
License: Ruby
Files: lib/metasm.rb lib/metasm/* data/cpuinfo/*
Copyright: 2006-2010 Yoann GUILLOT
License: LGPL-2.1

433
lib/fastlib.rb
View File

@ -1,433 +0,0 @@
#!/usr/bin/env ruby
# -*- coding: binary -*-
#
# FASTLIB is a mechanism for loading large sets of libraries in a way that is
# faster and much more flexible than typical disk structures. FASTLIB includes
# hooks that can be used for both compression and encoding of Ruby libraries.
#
#
# This format was specifically created to improve the performance and
# AV-resistance of the Metasploit Framework and Rex libraries.
#
#
# This library is still in its early form; a large number of performance and
# compatiblity improvements are not yet included. Do not depend on the FASTLIB
# file format at this time.
#
require "find"
#
# Copyright (C) 2011 Rapid7. You can redistribute it and/or
# modify it under the terms of the ruby license.
#
#
# Roughly based on the rubyzip zip/ziprequire library:
# >> Copyright (C) 2002 Thomas Sondergaard
# >> rubyzip is free software; you can redistribute it and/or
# >> modify it under the terms of the ruby license.
#
# The FastLib class implements the meat of the FASTLIB archive format
#
class FastLib
VERSION = "0.0.8"
FLAG_COMPRESS = 0x01
FLAG_ENCRYPT = 0x02
@@cache = {}
@@has_zlib = false
#
# Load zlib support if possible
#
begin
require 'zlib'
@@has_zlib = true
rescue ::LoadError
end
#
# This method returns the version of the fastlib library
#
def self.version
VERSION
end
#
# This method loads content from a specific archive file by name. If the
# noprocess argument is set to true, the contents will not be expanded to
# include workarounds for things such as __FILE__. This is useful when
# loading raw binary data where these strings may occur
#
def self.load(lib, name, noprocess=false)
data = ""
load_cache(lib)
return unless ( @@cache[lib] and @@cache[lib][name] )
::File.open(lib, "rb") do |fd|
fd.seek(
@@cache[lib][:fastlib_header][0] +
@@cache[lib][:fastlib_header][1] +
@@cache[lib][name][0]
)
data = fastlib_filter_decode( lib, fd.read(@@cache[lib][name][1] ))
end
# Return the contents in raw or processed form
noprocess ? data : post_process(lib, name, data)
end
#
# This method caches the file list and offsets within the archive
#
def self.load_cache(lib)
return if @@cache[lib]
@@cache[lib] = {}
return if not ::File.exists?(lib)
::File.open(lib, 'rb') do |fd|
dict = {}
head = fd.read(4)
return if head != "FAST"
hlen = fd.read(4).unpack("N")[0]
flag = fd.read(4).unpack("N")[0]
@@cache[lib][:fastlib_header] = [12, hlen, fd.stat.mtime.utc.to_i ]
@@cache[lib][:fastlib_flags] = flag
nlen, doff, dlen, tims = fd.read(16).unpack("N*")
while nlen > 0
name = fastlib_filter_decode( lib, fd.read(nlen) )
dict[name] = [doff, dlen, tims]
nlen, doff, dlen, tims = fd.read(16).unpack("N*")
end
@@cache[lib].merge!(dict)
end
end
#
# This method provides compression and encryption capabilities
# for the fastlib archive format.
#
def self.fastlib_filter_decode(lib, buff)
if (@@cache[lib][:fastlib_flags] & FLAG_ENCRYPT) != 0
@@cache[lib][:fastlib_decrypt] ||= ::Proc.new do |data|
stub = "decrypt_%.8x" % ( @@cache[lib][:fastlib_flags] & 0xfffffff0 )
FastLib.send(stub, data)
end
buff = @@cache[lib][:fastlib_decrypt].call( buff )
end
if (@@cache[lib][:fastlib_flags] & FLAG_COMPRESS) != 0
if not @@has_zlib
raise ::RuntimeError, "zlib is required to open this archive"
end
z = Zlib::Inflate.new
buff = z.inflate(buff)
buff << z.finish
z.close
end
buff
end
#
# This method provides compression and encryption capabilities
# for the fastlib archive format.
#
def self.fastlib_filter_encode(lib, buff)
if (@@cache[lib][:fastlib_flags] & FLAG_COMPRESS) != 0
if not @@has_zlib
raise ::RuntimeError, "zlib is required to open this archive"
end
z = Zlib::Deflate.new
buff = z.deflate(buff)
buff << z.finish
z.close
end
if (@@cache[lib][:fastlib_flags] & FLAG_ENCRYPT) != 0
@@cache[lib][:fastlib_encrypt] ||= ::Proc.new do |data|
stub = "encrypt_%.8x" % ( @@cache[lib][:fastlib_flags] & 0xfffffff0 )
FastLib.send(stub, data)
end
buff = @@cache[lib][:fastlib_encrypt].call( buff )
end
buff
end
# This method provides a way to create a FASTLIB archive programatically.
#
# @param [String] lib the output path for the archive
# @param [String] flag a string containing the hex values for the
# flags ({FLAG_COMPRESS} and {FLAG_ENCRYPT}).
# @param [String] bdir the path to the base directory which will be
# stripped from all paths included in the archive
# @param [Array<String>] dirs list of directories/files to pack into
# the archive. All dirs should be under bdir so that the paths are
# stripped correctly.
# @return [void]
def self.dump(lib, flag, bdir, *dirs)
head = ""
data = ""
hidx = 0
didx = 0
bdir = bdir.gsub(/\/$/, '')
brex = /^#{Regexp.escape(bdir)}\//
@@cache[lib] = {
:fastlib_flags => flag.to_i(16)
}
dirs.each do |dir|
::Find.find(dir) do |path|
next if not ::File.file?(path)
name = fastlib_filter_encode( lib, path.sub( brex, "" ) )
buff = ""
::File.open(path, "rb") do |fd|
buff = fastlib_filter_encode(lib, fd.read(fd.stat.size))
end
head << [ name.length, didx, buff.length, ::File.stat(path).mtime.utc.to_i ].pack("NNNN")
head << name
hidx = hidx + 16 + name.length
data << buff
didx = didx + buff.length
end
end
head << [0,0,0].pack("NNN")
::File.open(lib, "wb") do |fd|
fd.write("FAST")
fd.write( [ head.length, flag.to_i(16) ].pack("NN") )
fd.write( head )
fd.write( data )
end
end
#
# This archive provides a way to list the contents of an archive
# file, returning the names only in sorted order.
#
def self.list(lib)
load_cache(lib)
( @@cache[lib] || {} ).keys.map{|x| x.to_s }.sort.select{ |x| @@cache[lib][x] }
end
#
# This method is called on the loaded is required to expand __FILE__
# and other inline dynamic constants to map to the correct location.
#
def self.post_process(lib, name, data)
data.gsub('__FILE__', "'#{ ::File.expand_path(::File.join(::File.dirname(lib), name)) }'")
end
#
# This is a stub crypto handler that performs a basic XOR
# operation against a fixed one byte key. The two usable IDs
# are 12345600 and 00000000
#
def self.encrypt_12345600(data)
encrypt_00000000(data)
end
def self.decrypt_12345600(data)
encrypt_00000000(data)
end
def self.encrypt_00000000(data)
data.unpack("C*").map{ |c| c ^ 0x90 }.pack("C*")
end
def self.decrypt_00000000(data)
encrypt_00000000(data)
end
#
# Expose the cache to callers
#
def self.cache
@@cache
end
end
#
# Allow this library to be used as an executable to create and list
# FASTLIB archives
#
if __FILE__ == $0
cmd = ARGV.shift
unless ["store", "list", "version"].include?(cmd)
$stderr.puts "Usage: #{$0} [dump|list|version] <arguments>"
exit(0)
end
case cmd
when "store"
dst = ARGV.shift
flg = ARGV.shift
dir = ARGV.shift
src = ARGV
unless dst and dir and src.length > 0
$stderr.puts "Usage: #{$0} store destination.fastlib flags base_dir src1 src2 ... src99"
exit(0)
end
FastLib.dump(dst, flg, dir, *src)
when "list"
src = ARGV.shift
unless src
$stderr.puts "Usage: #{$0} list"
exit(0)
end
$stdout.puts "Library: #{src}"
$stdout.puts "====================================================="
FastLib.list(src).each do |name|
fsize = FastLib.cache[src][name][1]
ftime = ::Time.at(FastLib.cache[src][name][2]).strftime("%Y-%m-%d %H:%M:%S")
$stdout.puts sprintf("%9d\t%20s\t%s\n", fsize, ftime, name)
end
$stdout.puts ""
when "version"
$stdout.puts "FastLib Version #{FastLib.version}"
end
exit(0)
end
#
# FASTLIB archive format (subject to change without notice)
#
=begin
* All integers are 32-bit and in network byte order (big endian / BE)
* The file signature is 0x46415354 (big endian, use htonl() if necessary)
* The header is always 12 bytes into the archive (magic + header length)
* The data section is always 12 + header length into the archive
* The header entries always start with 'fastlib_header'
* The header entries always consist of 16 bytes + name length (no alignment)
* The header name data may be encoded, compressed, or transformed
* The data entries may be encoded, compressed, or transformed too
4 bytes: "FAST"
4 bytes: NBO header length
4 bytes: NBO flags (24-bit crypto ID, 8 bit modes)
[
4 bytes: name length (0 = End of Names)
4 bytes: data offset
4 bytes: data length
4 bytes: timestamp
]
[ Raw Data ]
=end
module Kernel #:nodoc:all
alias :fastlib_original_require :require
#
# Store the CWD when were initially loaded
# required for resolving relative paths
#
@@fastlib_base_cwd = ::Dir.pwd
#
# This method hooks the original Kernel.require to support
# loading files within FASTLIB archives
#
def require(name)
fastlib_require(name) || fastlib_original_require(name)
end
#
# This method handles the loading of FASTLIB archives
#
def fastlib_require(name)
if name.respond_to? :to_path
name = name.to_path
end
name = name + ".rb" if not name =~ /\.rb$/
return false if fastlib_already_loaded?(name)
return false if fastlib_already_tried?(name)
# XXX Implement relative search paths within archives
$:.map{ |path|
(path =~ /^([A-Za-z]\:|\/)/ ) ? path : ::File.expand_path( ::File.join(@@fastlib_base_cwd, path) )
}.map{ |path| ::Dir["#{path}/*.fastlib"] }.flatten.uniq.each do |lib|
data = FastLib.load(lib, name)
next if not data
$" << name
Object.class_eval(data, lib + "::" + name)
return true
end
$fastlib_miss << name
false
end
#
# This method determines whether the specific file name
# has already been loaded ($LOADED_FEATURES aka $")
#
def fastlib_already_loaded?(name)
re = Regexp.new("^" + Regexp.escape(name) + "$")
$".detect { |e| e =~ re } != nil
end
#
# This method determines whether the specific file name
# has already been attempted with the included FASTLIB
# archives.
#
# TODO: Ensure that this only applies to known FASTLIB
# archives and that newly included archives will
# be searched appropriately.
#
def fastlib_already_tried?(name)
$fastlib_miss ||= []
$fastlib_miss.include?(name)
end
end

1
lib/msf/core/module_manager.rb
View File

@ -7,7 +7,6 @@ require 'pathname'
#
# Project
#
require 'fastlib'
require 'msf/core'
require 'msf/core/module_set'

4
lib/msf/core/module_manager/loading.rb
View File

@ -7,7 +7,6 @@ require 'active_support/concern'
#
# Project
#
require 'msf/core/modules/loader/archive'
require 'msf/core/modules/loader/directory'
# Deals with loading modules for the {Msf::ModuleManager}
@ -20,7 +19,6 @@ module Msf::ModuleManager::Loading
# Classes that can be used to load modules.
LOADER_CLASSES = [
Msf::Modules::Loader::Archive,
Msf::Modules::Loader::Directory
]
@ -103,7 +101,7 @@ module Msf::ModuleManager::Loading
# Load all of the modules from the supplied directory or archive
#
# @param [String] path Path to a directory or Fastlib archive
# @param [String] path Path to a directory
# @param [Hash] options
# @option options [Boolean] :force Whether the force loading the modules even if they are unchanged and already
# loaded.

26
lib/msf/core/module_manager/module_paths.rb
View File

@ -24,30 +24,14 @@ module Msf::ModuleManager::ModulePaths
# Make the path completely canonical
pathname = Pathname.new(path_without_trailing_file_separator).expand_path
extension = pathname.extname
if extension == Msf::Modules::Loader::Archive::ARCHIVE_EXTENSION
unless pathname.exist?
raise ArgumentError, "The path supplied does not exist", caller
end
nested_paths << pathname.to_s
else
# Make sure the path is a valid directory
unless pathname.directory?
raise ArgumentError, "The path supplied is not a valid directory.", caller
end
nested_paths << pathname.to_s
# Identify any fastlib archives inside of this path
fastlib_glob = pathname.join('**', "*#{Msf::Modules::Loader::Archive::ARCHIVE_EXTENSION}")
Dir.glob(fastlib_glob).each do |fastlib_path|
nested_paths << fastlib_path
end
# Make sure the path is a valid directory
unless pathname.directory?
raise ArgumentError, "The path supplied is not a valid directory.", caller
end
nested_paths << pathname.to_s
# Update the module paths appropriately
self.module_paths = (module_paths + nested_paths).flatten.uniq

1
lib/msf/core/module_set.rb
View File

@ -1,6 +1,5 @@
# -*- coding: binary -*-
require 'msf/core'
require 'fastlib'
require 'pathname'
#

91
lib/msf/core/modules/loader/archive.rb
View File

@ -1,91 +0,0 @@
# -*- coding: binary -*-
require 'msf/core/modules/loader/base'
# Concerns loading modules form fastlib archives
class Msf::Modules::Loader::Archive < Msf::Modules::Loader::Base
#
# CONSTANTS
#
# The extension for Fastlib archives.
ARCHIVE_EXTENSION = '.fastlib'
# Returns true if the path is a Fastlib archive.
#
# @param (see Msf::Modules::Loader::Base#loadable?)
# @return [true] if path has the {ARCHIVE_EXTENSION} extname.
# @return [false] otherwise
def loadable?(path)
if File.extname(path) == ARCHIVE_EXTENSION
true
else
false
end
end
protected
# Yields the module_reference_name for each module file in the Fastlib archive at path.
#
# @param path [String] The path to the Fastlib archive file.
# @param opts [Hash] Additional options
# @yield (see Msf::Modules::Loader::Base#each_module_reference_name)
# @yieldparam (see Msf::Modules::Loader::Base#each_module_reference_name)
# @return (see Msf::Modules::Loader::Base#each_module_reference_name)
def each_module_reference_name(path, opts={})
whitelist = opts[:whitelist] || []
entries = ::FastLib.list(path)
entries.each do |entry|
if entry.include?('.svn/')
next
end
type = entry.split('/', 2)[0]
type = type.singularize
unless module_manager.type_enabled?(type)
next
end
if whitelist.empty?
if module_path?(entry)
# The module_reference_name doesn't have a file extension
module_reference_name = module_reference_name_from_path(entry)
yield path, type, module_reference_name
end
else
whitelist.each do |pattern|
if entry =~ pattern
yield path, type, module_reference_name
else
next
end
end
end
end
end
# Returns the path to the module inside the Fastlib archive. The path to the archive is separated from the path to
# the file inside the archive by '::'.
#
# @param (see Msf::Modules::Loader::Base#module_path)
# @return [String] Path to module file inside the Fastlib archive.
def module_path(parent_path, type, module_reference_name)
file_path = typed_path(type, module_reference_name)
module_path = "#{parent_path}::#{file_path}"
module_path
end
# Loads the module content from the Fastlib archive.
#
# @return (see Msf::Modules::Loader::Base#read_module_content)
def read_module_content(path, type, module_reference_name)
file_path = typed_path(type, module_reference_name)
::FastLib.load(path, file_path)
end
end

4
lib/msf/core/modules/loader/directory.rb
View File

@ -1,4 +1,8 @@
# -*- coding: binary -*-
require 'msf/core/modules/loader'
require 'msf/core/modules/loader/base'
# Concerns loading module from a directory
class Msf::Modules::Loader::Directory < Msf::Modules::Loader::Base
# Returns true if the path is a directory

1
msfbinscan
View File

@ -11,7 +11,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfcli
View File

@ -535,7 +535,6 @@ class Msfcli
end
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']
require 'fastlib'
require 'msfenv'
require 'msf/ui'
require 'msf/base'

1
msfd
View File

@ -17,7 +17,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfelfscan
View File

@ -11,7 +11,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfencode
View File

@ -11,7 +11,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfmachscan
View File

@ -11,7 +11,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfpayload
View File

@ -11,7 +11,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfpescan
View File

@ -11,7 +11,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfrop
View File

@ -14,7 +14,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfrpc
View File

@ -15,7 +15,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfrpcd
View File

@ -15,7 +15,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'

1
msfvenom
View File

@ -7,7 +7,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

234
spec/lib/fastlib_spec.rb
View File

@ -1,234 +0,0 @@
# -*- coding:binary -*-
require 'spec_helper'
require 'msf/core'
describe FastLib do
let(:archived_paths) do
[
File.join('auxiliary', 'scanner', 'portscan', 'xmas.rb'),
File.join('exploits', 'windows', 'smb', 'ms08_067_netapi.rb')
]
end
let(:base_path) do
File.join(Msf::Config.install_root, 'modules')
end
let(:extension) do
'.fastlib'
end
let(:flag_compress) do
0x01
end
let(:flag_encrypt) do
0x02
end
let(:unarchived_paths) do
archived_paths.collect { |archived_path|
File.join(base_path, archived_path)
}
end
context 'CONSTANTS' do
context 'flags' do
it 'should have compression' do
described_class::FLAG_COMPRESS.should == flag_compress
end
it 'should have encryption' do
described_class::FLAG_ENCRYPT.should == flag_encrypt
end
end
end
context 'class methods' do
context 'dump' do
let(:flag_string) do
flags.to_s(16)
end
before(:each) do
FastLib.cache.clear
end
around(:each) do |example|
Dir.mktmpdir do |directory|
@destination_path = File.join(directory, "rspec#{extension}")
example.run
end
end
context 'without compression and without encryption' do
let(:flags) do
0x0
end
it 'should create an archive' do
File.exist?(@destination_path).should be_falsey
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
File.exist?(@destination_path).should be_truthy
end
context 'cache' do
it 'should populate' do
FastLib.cache[@destination_path].should be_nil
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
FastLib.cache[@destination_path].should be_a Hash
end
it 'should include flags' do
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
FastLib.cache[@destination_path][:fastlib_flags].should == flags
end
skip "Fix https://www.pivotaltracker.com/story/show/38730815" do
it 'should include header' do
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
header = FastLib.cache[@destination_path][:fastlib_header]
modification_time = File.mtime(@destination_path).utc.to_i
header.should be_a Array
# @todo figure out why 12 before header length
header[0].should == 12
# @todo figure out why header length is 0
header[1].should == 0
header[2].should == modification_time
end
it 'should include archived paths' do
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
cache = FastLib.cache[@destination_path]
archived_path = File.join('exploits', 'windows', 'smb', 'ms08_067_netapi.rb')
unarchived_path = File.join(base_path, archived_path)
# make sure that the unarchived module exists and hasn't be deleted or renamed before expecting it to be
# in the archive.
File.exist?(unarchived_path).should be_truthy
cache[archived_path].should_not be_nil
end
end
end
end
context 'with compression and without encryption' do
let(:flags) do
flag_compress
end
it 'should create an archive' do
File.exist?(@destination_path).should be_falsey
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
File.exist?(@destination_path).should be_truthy
end
it 'should be smaller than the uncompressed archive' do
uncompressed_path = "#{@destination_path}.uncompressed"
compressed_path = "#{@destination_path}.compressed"
File.exist?(uncompressed_path).should be_falsey
File.exist?(compressed_path).should be_falsey
described_class.dump(uncompressed_path, '', base_path, *unarchived_paths)
described_class.dump(compressed_path, flag_string, base_path, *unarchived_paths)
File.exist?(uncompressed_path).should be_truthy
File.exist?(compressed_path).should be_truthy
File.size(compressed_path).should < File.size(uncompressed_path)
end
end
context 'without compression and with encryption' do
let(:flags) do
flag_encrypt
end
it 'should create an archive' do
File.exist?(@destination_path).should be_falsey
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
File.exist?(@destination_path).should be_truthy
end
end
context 'with compression and with encryption' do
let(:flags) do
flag_compress | flag_encrypt
end
it 'should create an archive' do
File.exist?(@destination_path).should be_falsey
described_class.dump(@destination_path, flag_string, base_path, *unarchived_paths)
File.exist?(@destination_path).should be_truthy
end
end
end
context 'list' do
around(:each) do |example|
Dir.mktmpdir do |directory|
@destination_path = File.join(directory, "rspec#{extension}")
FastLib.dump(@destination_path, FastLib::FLAG_COMPRESS.to_s, base_path, *unarchived_paths)
example.run
end
end
# ensure modules expected to be listed actually exist
it 'should use existent unarchived modules' do
unarchived_paths.each do |unarchived_path|
File.exist?(unarchived_path).should be_truthy
end
end
skip "Fix https://www.pivotaltracker.com/story/show/38730815" do
context 'with cached dump' do
it 'should have dump cached' do
FastLib.cache[@destination_path].should_not be_nil
end
it 'should list archived paths' do
paths = FastLib.list(@destination_path)
paths.length.should == archived_paths.length
paths.should == archived_paths
end
end
end
context 'without cached dump' do
before(:each) do
FastLib.cache.clear
end
it 'should not have dump cache' do
FastLib.cache[@destination_path].should be_nil
end
it 'should list archived paths' do
paths = FastLib.list(@destination_path)
paths.length.should == archived_paths.length
paths.should == archived_paths
end
end
end
end
end

8
spec/lib/msf/core/module_manager_spec.rb
View File

@ -19,14 +19,6 @@ require 'msf/core'
describe Msf::ModuleManager do
include_context 'Msf::Simple::Framework'
let(:archive_basename) do
[basename_prefix, archive_extension]
end
let(:archive_extension) do
'.fastlib'
end
let(:basename_prefix) do
'rspec'
end

276
spec/lib/msf/core/modules/loader/archive_spec.rb
View File

@ -1,276 +0,0 @@
# -*- coding:binary -*-
require 'spec_helper'
require 'msf/core'
describe Msf::Modules::Loader::Archive do
let(:archive_extension) do
'.fastlib'
end
context 'CONSTANTS' do
it 'should have extension' do
described_class::ARCHIVE_EXTENSION.should == archive_extension
end
end
context 'instance methods' do
let(:enabled_type) do
'exploit'
end
let(:enabled_type_directory) do
'exploits'
end
let(:framework) do
double('Framework')
end
let(:module_extension) do
'.rb'
end
let(:module_manager) do
# DO NOT mock module_manager to ensure that no protected methods are being called.
Msf::ModuleManager.new(framework, [enabled_type])
end
let(:module_reference_name) do
'module/reference/name'
end
subject do
described_class.new(module_manager)
end
context '#each_module_reference_name' do
let(:disabled_module_content) do
<<-EOS
class Metasploit3 < Msf::Auxiliary
end
EOS
end
let(:disabled_type) do
'auxiliary'
end
let(:disabled_type_directory) do
'auxiliary'
end
let(:enabled_module_content) do
<<-EOS
class Metasploit3 < Msf::Exploit::Remote
end
EOS
end
around(:each) do |example|
Dir.mktmpdir do |directory|
@base_path = directory
# make a .svn directory to be ignored
subversion_path = File.join(@base_path, '.svn')
FileUtils.mkdir_p subversion_path
# make a type directory that should be ignored because it's not enabled
disabled_type_path = File.join(@base_path, disabled_type_directory)
FileUtils.mkdir_p disabled_type_path
#
# create a valid module in the disabled type directory to make sure it's the enablement that's preventing the
# yield
#
disabled_module_path = File.join(disabled_type_path, "#{disabled_type}#{module_extension}")
File.open(disabled_module_path, 'wb') do |f|
f.write(disabled_module_content)
end
# make a type directory that should not be ignored because it is enabled
enabled_module_path = File.join(
@base_path,
enabled_type_directory,
"#{module_reference_name}#{module_extension}"
)
enabled_module_directory = File.dirname(enabled_module_path)
FileUtils.mkdir_p enabled_module_directory
File.open(enabled_module_path, 'wb') do |f|
f.write(enabled_module_content)
end
Dir.mktmpdir do |archive_directory|
@archive_path = File.join(archive_directory, "rspec#{archive_extension}")
FastLib.dump(@archive_path, FastLib::FLAG_COMPRESS.to_s(16), @base_path, @base_path)
# @todo Fix https://www.pivotaltracker.com/story/show/38730815 and the cache won't need to be cleared as a work-around
FastLib.cache.clear
example.run
end
end
end
# this checks that the around(:each) is working
it 'should have an existent FastLib' do
File.exist?(@archive_path).should be_truthy
end
it 'should ignore .svn directories' do
subject.send(:each_module_reference_name, @archive_path) do |parent_path, type, module_reference_name|
parent_path.should_not include('.svn')
end
end
it 'should ignore types that are not enabled' do
module_manager.type_enabled?(disabled_type).should be_falsey
subject.send(:each_module_reference_name, @archive_path) do |parent_path, type, module_reference_name|
type.should_not == disabled_type
end
end
it 'should yield (parent_path, type, module_reference_name) with parent_path equal to the archive path' do
subject.send(:each_module_reference_name, @archive_path) do |parent_path, type, module_reference_name|
parent_path.should == @archive_path
end
end
it 'should yield (parent_path, type, module_reference_name) with type equal to enabled type' do
module_manager.type_enabled?(enabled_type).should be_truthy
subject.send(:each_module_reference_name, @archive_path) do |parent_path, type, module_reference_name|
type.should == enabled_type
end
end
it 'should yield (path, type, module_reference_name) with module_reference_name without extension' do
subject.send(:each_module_reference_name, @archive_path) do |parent_path, type, module_reference_name|
module_reference_name.should_not match(/#{Regexp.escape(module_extension)}$/)
module_reference_name.should == module_reference_name
end
end
# ensure that the block is actually being run so that shoulds in the block aren't just being skipped
it 'should yield the correct number of tuples' do
actual_count = 0
subject.send(:each_module_reference_name, @archive_path) do |parent_path, type, module_reference_name|
actual_count += 1
end
actual_count.should == 1
end
end
context '#loadable?' do
it 'should return true if the path has ARCHIVE_EXTENSION as file extension' do
path = "path/to/archive#{archive_extension}"
File.extname(path).should == described_class::ARCHIVE_EXTENSION
subject.loadable?(path).should be_truthy
end
it 'should return false if the path contains ARCHIVE_EXTENSION, but it is not the file extension' do
path = "path/to/archive#{archive_extension}.bak"
path.should include(described_class::ARCHIVE_EXTENSION)
File.extname(path).should_not == described_class::ARCHIVE_EXTENSION
subject.loadable?(path).should be_falsey
end
end
context '#module_path' do
let(:parent_path) do
"path/to/archive#{archive_extension}"
end
let(:type) do
'exploit'
end
let(:type_directory) do
'exploits'
end
it 'should use typed_path to convert the type name to a type directory' do
subject.should_receive(:typed_path).with(type, module_reference_name)
subject.send(:module_path, parent_path, type, module_reference_name)
end
it "should separate the archive path from the entry path with '::'" do
module_path = subject.send(:module_path, parent_path, type, module_reference_name)
module_path.should == "#{parent_path}::#{type_directory}/#{module_reference_name}.rb"
end
end
context '#read_module_path' do
let(:module_reference_name) do
'windows/smb/ms08_067_netapi'
end
let(:type) do
enabled_type
end
let(:type_directory) do
enabled_type_directory
end
let(:archived_path) do
File.join(type_directory, "#{module_reference_name}#{module_extension}")
end
let(:base_path) do
File.join(Msf::Config.install_root, 'modules')
end
let(:flag_string) do
flags.to_s(16)
end
let(:flags) do
0x0
end
let(:unarchived_path) do
File.join(base_path, archived_path)
end
it 'should read modules that exist' do
File.exist?(unarchived_path).should be_truthy
end
around(:each) do |example|
Dir.mktmpdir do |directory|
@parent_path = File.join(directory, 'rspec.fastlib')
FastLib.dump(@parent_path, flag_string, base_path, unarchived_path)
# @todo Fix https://www.pivotaltracker.com/story/show/38730815 so cache from dump is correct
FastLib.cache.clear
example.run
end
end
context 'with uncompressed archive' do
it_should_behave_like 'Msf::Modules::Loader::Archive#read_module_content'
end
context 'with compressed archive' do
let(:flags) do
FastLib::FLAG_COMPRESS
end
it_should_behave_like 'Msf::Modules::Loader::Archive#read_module_content'
end
end
end
end

1
spec/lib/rex/proto/pjl/client_spec.rb
View File

@ -1,5 +1,4 @@
require 'spec_helper'
require 'fastlib'
require 'msfenv'
require 'msf/base'
require 'rex/proto/pjl'

1
spec/msfcli_spec.rb
View File

@ -2,7 +2,6 @@ require 'spec_helper'
load Metasploit::Framework.root.join('msfcli').to_path
require 'fastlib'
require 'msfenv'
require 'msf/ui'
require 'msf/base'

40
spec/support/shared/examples/msf/module_manager/module_paths.rb
View File

@ -14,33 +14,6 @@ shared_examples_for 'Msf::ModuleManager::ModulePaths' do
end
end
context 'with Fastlib archive' do
it 'should raise an ArgumentError unless the File exists' do
file = Tempfile.new(archive_basename)
# unlink will clear path, so copy it to a variable
path = file.path
file.unlink
File.exist?(path).should be_falsey
expect {
module_manager.add_module_path(path)
}.to raise_error(ArgumentError, "The path supplied does not exist")
end
it 'should add the path to #module_paths if the File exists' do
Tempfile.open(archive_basename) do |temporary_file|
path = temporary_file.path
File.exist?(path).should be_truthy
module_manager.add_module_path(path)
module_paths.should include(path)
end
end
end
context 'with directory' do
it 'should add path to #module_paths' do
Dir.mktmpdir do |path|
@ -49,19 +22,6 @@ shared_examples_for 'Msf::ModuleManager::ModulePaths' do
module_paths.should include(path)
end
end
context 'containing Fastlib archives' do
it 'should add each Fastlib archive to #module_paths' do
Dir.mktmpdir do |directory|
Tempfile.open(archive_basename, directory) do |file|
module_manager.add_module_path(directory)
module_paths.should include(directory)
module_paths.should include(file.path)
end
end
end
end
end
context 'with other file' do

14
spec/support/shared/examples/msf/modules/loader_archive_read_module_content.rb
View File

@ -1,14 +0,0 @@
# -*- coding:binary -*-
shared_examples_for 'Msf::Modules::Loader::Archive#read_module_content' do
it 'should be able to read the module content' do
archived_module_content = subject.send(:read_module_content, @parent_path, type, module_reference_name)
unarchived_module_content = ''
File.open(unarchived_path) do |f|
unarchived_module_content = f.read
end
unarchived_module_content.should_not be_empty
archived_module_content.should == unarchived_module_content
end
end

1
spec/tools/cpassword_decrypt_spec.rb
View File

@ -2,7 +2,6 @@ require 'spec_helper'
load Metasploit::Framework.root.join('tools/cpassword_decrypt.rb').to_path
require 'fastlib'
require 'msfenv'
require 'msf/base'

1
spec/tools/virustotal_spec.rb
View File

@ -2,7 +2,6 @@ require 'spec_helper'
load Metasploit::Framework.root.join('tools/virustotal.rb').to_path
require 'fastlib'
require 'msfenv'
require 'msf/base'
require 'digest/sha2'

1
test/tests/test_encoders.rb
View File

@ -11,7 +11,6 @@ end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', '..', 'lib')))
require 'fastlib'
require 'msfenv'
require 'msf/base'

1
tools/cpassword_decrypt.rb
View File

@ -38,7 +38,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
require 'rex'

1
tools/exe2vba.rb
View File

@ -14,7 +14,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/exe2vbs.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/find_badchars.rb
View File

@ -14,7 +14,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/halflm_second.rb
View File

@ -16,7 +16,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/hmac_sha1_crack.rb
View File

@ -16,7 +16,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/list_interfaces.rb
View File

@ -15,7 +15,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/lm2ntcrack.rb
View File

@ -14,7 +14,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/metasm_shell.rb
View File

@ -21,7 +21,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_author.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_changelog.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_count.rb
View File

@ -8,7 +8,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_disclodate.rb
View File

@ -12,7 +12,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_license.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_mixins.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_payloads.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_ports.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_rank.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_reference.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/module_targets.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/msf_irb_shell.rb
View File

@ -10,7 +10,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/nasm_shell.rb
View File

@ -15,7 +15,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

6
tools/pack_fastlib.sh
View File

@ -1,6 +0,0 @@
#!/bin/sh
mkdir fastlib-archived
./lib/fastlib.rb store modules.fastlib 12345603 modules/ modules/*
./lib/fastlib.rb store lib/metasploit.fastlib 12345603 lib lib/msf/ lib/rex*
mv lib/msf lib/rex* modules/ fastlib-archived

1
tools/pattern_create.rb
View File

@ -10,7 +10,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/pattern_offset.rb
View File

@ -8,7 +8,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/payload_lengths.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/psexec.rb
View File

@ -10,7 +10,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/reg.rb
View File

@ -13,7 +13,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
$:.unshift(ENV['MSF_LOCAL_LIB']) if ENV['MSF_LOCAL_LIB']

1
tools/virustotal.rb
View File

@ -34,7 +34,6 @@ while File.symlink?(msfbase)
end
$:.unshift(File.expand_path(File.join(File.dirname(msfbase), '..', 'lib')))
require 'fastlib'
require 'msfenv'
require 'rex'
require 'msf/core'