github
/
NewPipe
mirror of https://github.com/TeamNewPipe/NewPipe
1
Fork 0
Code Releases Activity

Merge branch 'dev'

This commit is contained in:
Stypox 2024-04-23 19:23:20 +02:00
parent a557ac3c7b a69bbab732
commit 24e65ef018
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23
27 changed files with 666 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/src/main/java/org/schabi/newpipe/error/UserAction.java
View File

@ -6,6 +6,7 @@ package org.schabi.newpipe.error;
public enum UserAction {
USER_REPORT("user report"),
UI_ERROR("ui error"),
DATABASE_IMPORT_EXPORT("database import or export"),
SUBSCRIPTION_CHANGE("subscription change"),
SUBSCRIPTION_UPDATE("subscription update"),
SUBSCRIPTION_GET("get subscription"),

46
app/src/main/java/org/schabi/newpipe/settings/BackupRestoreSettingsFragment.java
View File

@ -21,9 +21,15 @@ import androidx.core.content.ContextCompat;
import androidx.preference.Preference;
import androidx.preference.PreferenceManager;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.NewPipeDatabase;
import org.schabi.newpipe.R;
import org.schabi.newpipe.error.ErrorInfo;
import org.schabi.newpipe.error.ErrorUtil;
import org.schabi.newpipe.error.UserAction;
import org.schabi.newpipe.settings.export.BackupFileLocator;
import org.schabi.newpipe.settings.export.ImportExportManager;
import org.schabi.newpipe.streams.io.NoFileManagerSafeGuard;
import org.schabi.newpipe.streams.io.StoredFileHelper;
import org.schabi.newpipe.util.NavigationHelper;
@ -42,7 +48,7 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
private final SimpleDateFormat exportDateFormat =
new SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US);
private ContentSettingsManager manager;
private ImportExportManager manager;
private String importExportDataPathKey;
private final ActivityResultLauncher<Intent> requestImportPathLauncher =
registerForActivityResult(new ActivityResultContracts.StartActivityForResult(),
@ -57,8 +63,7 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
@Nullable final String rootKey) {
final File homeDir = ContextCompat.getDataDir(requireContext());
Objects.requireNonNull(homeDir);
manager = new ContentSettingsManager(new NewPipeFileLocator(homeDir));
manager.deleteSettingsFile();
manager = new ImportExportManager(new BackupFileLocator(homeDir));
importExportDataPathKey = getString(R.string.import_export_data_path);
@ -165,7 +170,7 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
Toast.makeText(requireContext(), R.string.export_complete_toast, Toast.LENGTH_SHORT)
.show();
} catch (final Exception e) {
ErrorUtil.showUiErrorSnackbar(this, "Exporting database", e);
showErrorSnackbar(e, "Exporting database and settings");
}
}
@ -182,6 +187,7 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
throw new IOException("Could not create databases dir");
}
// replace the current database
if (!manager.extractDb(file)) {
Toast.makeText(requireContext(), R.string.could_not_import_all_files,
Toast.LENGTH_LONG)
@ -189,9 +195,13 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
}
// if settings file exist, ask if it should be imported.
if (manager.extractSettings(file)) {
final boolean hasJsonPrefs = manager.exportHasJsonPrefs(file);
if (hasJsonPrefs || manager.exportHasSerializedPrefs(file)) {
new androidx.appcompat.app.AlertDialog.Builder(requireContext())
.setTitle(R.string.import_settings)
.setMessage(hasJsonPrefs ? null : requireContext()
.getString(R.string.import_settings_vulnerable_format))
.setOnDismissListener(dialog -> finishImport(importDataUri))
.setNegativeButton(R.string.cancel, (dialog, which) -> {
dialog.dismiss();
finishImport(importDataUri);
@ -201,7 +211,16 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
final Context context = requireContext();
final SharedPreferences prefs = PreferenceManager
.getDefaultSharedPreferences(context);
manager.loadSharedPreferences(prefs);
try {
if (hasJsonPrefs) {
manager.loadJsonPrefs(file, prefs);
} else {
manager.loadSerializedPrefs(file, prefs);
}
} catch (IOException | ClassNotFoundException | JsonParserException e) {
createErrorNotification(e, "Importing preferences");
return;
}
cleanImport(context, prefs);
finishImport(importDataUri);
})
@ -210,7 +229,7 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
finishImport(importDataUri);
}
} catch (final Exception e) {
ErrorUtil.showUiErrorSnackbar(this, "Importing database", e);
showErrorSnackbar(e, "Importing database and settings");
}
}
@ -247,7 +266,7 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
}
/**
* Save import path and restart system.
* Save import path and restart app.
*
* @param importDataUri The import path to save
*/
@ -268,4 +287,15 @@ public class BackupRestoreSettingsFragment extends BasePreferenceFragment {
.putString(importExportDataPathKey, importExportDataUri.toString());
editor.apply();
}
private void showErrorSnackbar(final Throwable e, final String request) {
ErrorUtil.showSnackbar(this, new ErrorInfo(e, UserAction.DATABASE_IMPORT_EXPORT, request));
}
private void createErrorNotification(final Throwable e, final String request) {
ErrorUtil.createNotification(
requireContext(),
new ErrorInfo(e, UserAction.DATABASE_IMPORT_EXPORT, request)
);
}
}

120
app/src/main/java/org/schabi/newpipe/settings/ContentSettingsManager.kt
View File

@ -1,120 +0,0 @@
package org.schabi.newpipe.settings
import android.content.SharedPreferences
import android.util.Log
import org.schabi.newpipe.MainActivity.DEBUG
import org.schabi.newpipe.streams.io.SharpOutputStream
import org.schabi.newpipe.streams.io.StoredFileHelper
import org.schabi.newpipe.util.ZipHelper
import java.io.IOException
import java.io.ObjectInputStream
import java.io.ObjectOutputStream
import java.util.zip.ZipOutputStream
class ContentSettingsManager(private val fileLocator: NewPipeFileLocator) {
companion object {
const val TAG = "ContentSetManager"
}
/**
* Exports given [SharedPreferences] to the file in given outputPath.
* It also creates the file.
*/
@Throws(Exception::class)
fun exportDatabase(preferences: SharedPreferences, file: StoredFileHelper) {
file.create()
ZipOutputStream(SharpOutputStream(file.stream).buffered())
.use { outZip ->
ZipHelper.addFileToZip(outZip, fileLocator.db.path, "newpipe.db")
try {
ObjectOutputStream(fileLocator.settings.outputStream()).use { output ->
output.writeObject(preferences.all)
output.flush()
}
} catch (e: IOException) {
if (DEBUG) {
Log.e(TAG, "Unable to exportDatabase", e)
}
}
ZipHelper.addFileToZip(outZip, fileLocator.settings.path, "newpipe.settings")
}
}
fun deleteSettingsFile() {
fileLocator.settings.delete()
}
/**
* Tries to create database directory if it does not exist.
*
* @return Whether the directory exists afterwards.
*/
fun ensureDbDirectoryExists(): Boolean {
return fileLocator.dbDir.exists() || fileLocator.dbDir.mkdir()
}
fun extractDb(file: StoredFileHelper): Boolean {
val success = ZipHelper.extractFileFromZip(file, fileLocator.db.path, "newpipe.db")
if (success) {
fileLocator.dbJournal.delete()
fileLocator.dbWal.delete()
fileLocator.dbShm.delete()
}
return success
}
fun extractSettings(file: StoredFileHelper): Boolean {
return ZipHelper.extractFileFromZip(file, fileLocator.settings.path, "newpipe.settings")
}
/**
* Remove all shared preferences from the app and load the preferences supplied to the manager.
*/
fun loadSharedPreferences(preferences: SharedPreferences) {
try {
val preferenceEditor = preferences.edit()
ObjectInputStream(fileLocator.settings.inputStream()).use { input ->
preferenceEditor.clear()
@Suppress("UNCHECKED_CAST")
val entries = input.readObject() as Map<String, *>
for ((key, value) in entries) {
when (value) {
is Boolean -> {
preferenceEditor.putBoolean(key, value)
}
is Float -> {
preferenceEditor.putFloat(key, value)
}
is Int -> {
preferenceEditor.putInt(key, value)
}
is Long -> {
preferenceEditor.putLong(key, value)
}
is String -> {
preferenceEditor.putString(key, value)
}
is Set<*> -> {
// There are currently only Sets with type String possible
@Suppress("UNCHECKED_CAST")
preferenceEditor.putStringSet(key, value as Set<String>?)
}
}
}
preferenceEditor.commit()
}
} catch (e: IOException) {
if (DEBUG) {
Log.e(TAG, "Unable to loadSharedPreferences", e)
}
} catch (e: ClassNotFoundException) {
if (DEBUG) {
Log.e(TAG, "Unable to loadSharedPreferences", e)
}
}
}
}

21
app/src/main/java/org/schabi/newpipe/settings/NewPipeFileLocator.kt
View File

@ -1,21 +0,0 @@
package org.schabi.newpipe.settings
import java.io.File
/**
* Locates specific files of NewPipe based on the home directory of the app.
*/
class NewPipeFileLocator(private val homeDir: File) {
val dbDir by lazy { File(homeDir, "/databases") }
val db by lazy { File(homeDir, "/databases/newpipe.db") }
val dbJournal by lazy { File(homeDir, "/databases/newpipe.db-journal") }
val dbShm by lazy { File(homeDir, "/databases/newpipe.db-shm") }
val dbWal by lazy { File(homeDir, "/databases/newpipe.db-wal") }
val settings by lazy { File(homeDir, "/databases/newpipe.settings") }
}

28
app/src/main/java/org/schabi/newpipe/settings/export/BackupFileLocator.kt Normal file
View File

@ -0,0 +1,28 @@
package org.schabi.newpipe.settings.export
import java.io.File
/**
* Locates specific files of NewPipe based on the home directory of the app.
*/
class BackupFileLocator(private val homeDir: File) {
companion object {
const val FILE_NAME_DB = "newpipe.db"
@Deprecated(
"Serializing preferences with Java's ObjectOutputStream is vulnerable to injections",
replaceWith = ReplaceWith("FILE_NAME_JSON_PREFS")
)
const val FILE_NAME_SERIALIZED_PREFS = "newpipe.settings"
const val FILE_NAME_JSON_PREFS = "preferences.json"
}
val dbDir by lazy { File(homeDir, "/databases") }
val db by lazy { File(dbDir, FILE_NAME_DB) }
val dbJournal by lazy { File(dbDir, "$FILE_NAME_DB-journal") }
val dbShm by lazy { File(dbDir, "$FILE_NAME_DB-shm") }
val dbWal by lazy { File(dbDir, "$FILE_NAME_DB-wal") }
}

180
app/src/main/java/org/schabi/newpipe/settings/export/ImportExportManager.kt Normal file
View File

@ -0,0 +1,180 @@
package org.schabi.newpipe.settings.export
import android.content.SharedPreferences
import com.grack.nanojson.JsonArray
import com.grack.nanojson.JsonParser
import com.grack.nanojson.JsonParserException
import com.grack.nanojson.JsonWriter
import org.schabi.newpipe.streams.io.SharpOutputStream
import org.schabi.newpipe.streams.io.StoredFileHelper
import org.schabi.newpipe.util.ZipHelper
import java.io.FileNotFoundException
import java.io.IOException
import java.io.ObjectOutputStream
import java.util.zip.ZipOutputStream
class ImportExportManager(private val fileLocator: BackupFileLocator) {
companion object {
const val TAG = "ImportExportManager"
}
/**
* Exports given [SharedPreferences] to the file in given outputPath.
* It also creates the file.
*/
@Throws(Exception::class)
fun exportDatabase(preferences: SharedPreferences, file: StoredFileHelper) {
file.create()
ZipOutputStream(SharpOutputStream(file.stream).buffered()).use { outZip ->
// add the database
ZipHelper.addFileToZip(
outZip,
BackupFileLocator.FILE_NAME_DB,
fileLocator.db.path,
)
// add the legacy vulnerable serialized preferences (will be removed in the future)
ZipHelper.addFileToZip(
outZip,
BackupFileLocator.FILE_NAME_SERIALIZED_PREFS
) { byteOutput ->
ObjectOutputStream(byteOutput).use { output ->
output.writeObject(preferences.all)
output.flush()
}
}
// add the JSON preferences
ZipHelper.addFileToZip(
outZip,
BackupFileLocator.FILE_NAME_JSON_PREFS
) { byteOutput ->
JsonWriter
.indent("")
.on(byteOutput)
.`object`(preferences.all)
.done()
}
}
}
/**
* Tries to create database directory if it does not exist.
*
* @return Whether the directory exists afterwards.
*/
fun ensureDbDirectoryExists(): Boolean {
return fileLocator.dbDir.exists() || fileLocator.dbDir.mkdir()
}
/**
* Extracts the database from the given file to the app's database directory.
* The current app's database will be overwritten.
* @param file the .zip file to extract the database from
* @return true if the database was successfully extracted, false otherwise
*/
fun extractDb(file: StoredFileHelper): Boolean {
val success = ZipHelper.extractFileFromZip(
file,
BackupFileLocator.FILE_NAME_DB,
fileLocator.db.path,
)
if (success) {
fileLocator.dbJournal.delete()
fileLocator.dbWal.delete()
fileLocator.dbShm.delete()
}
return success
}
@Deprecated(
"Serializing preferences with Java's ObjectOutputStream is vulnerable to injections",
replaceWith = ReplaceWith("exportHasJsonPrefs")
)
fun exportHasSerializedPrefs(zipFile: StoredFileHelper): Boolean {
return ZipHelper.zipContainsFile(zipFile, BackupFileLocator.FILE_NAME_SERIALIZED_PREFS)
}
fun exportHasJsonPrefs(zipFile: StoredFileHelper): Boolean {
return ZipHelper.zipContainsFile(zipFile, BackupFileLocator.FILE_NAME_JSON_PREFS)
}
/**
* Remove all shared preferences from the app and load the preferences supplied to the manager.
*/
@Deprecated(
"Serializing preferences with Java's ObjectOutputStream is vulnerable to injections",
replaceWith = ReplaceWith("loadJsonPrefs")
)
@Throws(IOException::class, ClassNotFoundException::class)
fun loadSerializedPrefs(zipFile: StoredFileHelper, preferences: SharedPreferences) {
ZipHelper.extractFileFromZip(zipFile, BackupFileLocator.FILE_NAME_SERIALIZED_PREFS) {
PreferencesObjectInputStream(it).use { input ->
@Suppress("UNCHECKED_CAST")
val entries = input.readObject() as Map<String, *>
val editor = preferences.edit()
editor.clear()
for ((key, value) in entries) {
when (value) {
is Boolean -> editor.putBoolean(key, value)
is Float -> editor.putFloat(key, value)
is Int -> editor.putInt(key, value)
is Long -> editor.putLong(key, value)
is String -> editor.putString(key, value)
is Set<*> -> {
// There are currently only Sets with type String possible
@Suppress("UNCHECKED_CAST")
editor.putStringSet(key, value as Set<String>?)
}
}
}
if (!editor.commit()) {
throw IOException("Unable to commit loadSerializedPrefs")
}
}
}.let { fileExists ->
if (!fileExists) {
throw FileNotFoundException(BackupFileLocator.FILE_NAME_SERIALIZED_PREFS)
}
}
}
/**
* Remove all shared preferences from the app and load the preferences supplied to the manager.
*/
@Throws(IOException::class, JsonParserException::class)
fun loadJsonPrefs(zipFile: StoredFileHelper, preferences: SharedPreferences) {
ZipHelper.extractFileFromZip(zipFile, BackupFileLocator.FILE_NAME_JSON_PREFS) {
val jsonObject = JsonParser.`object`().from(it)
val editor = preferences.edit()
editor.clear()
for ((key, value) in jsonObject) {
when (value) {
is Boolean -> editor.putBoolean(key, value)
is Float -> editor.putFloat(key, value)
is Int -> editor.putInt(key, value)
is Long -> editor.putLong(key, value)
is String -> editor.putString(key, value)
is JsonArray -> {
editor.putStringSet(key, value.mapNotNull { e -> e as? String }.toSet())
}
}
}
if (!editor.commit()) {
throw IOException("Unable to commit loadJsonPrefs")
}
}.let { fileExists ->
if (!fileExists) {
throw FileNotFoundException(BackupFileLocator.FILE_NAME_JSON_PREFS)
}
}
}
}

58
app/src/main/java/org/schabi/newpipe/settings/export/PreferencesObjectInputStream.java Normal file
View File

@ -0,0 +1,58 @@
package org.schabi.newpipe.settings.export;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectStreamClass;
import java.util.Set;
/**
* An {@link ObjectInputStream} that only allows preferences-related types to be deserialized, to
* prevent injections. The only allowed types are: all primitive types, all boxed primitive types,
* null, strings. HashMap, HashSet and arrays of previously defined types are also allowed. Sources:
* <a href="https://wiki.sei.cmu.edu/confluence/display/java/SER00-J.+Enable+serialization+compatibility+during+class+evolution">
* cmu.edu
* </a>,
* <a href="https://cheatsheetseries.owasp.org/cheatsheets/Deserialization_Cheat_Sheet.html#harden-your-own-javaioobjectinputstream">
* OWASP cheatsheet
* </a>,
* <a href="https://commons.apache.org/proper/commons-io/apidocs/src-html/org/apache/commons/io/serialization/ValidatingObjectInputStream.html#line-118">
* Apache's {@code ValidatingObjectInputStream}
* </a>
*/
public class PreferencesObjectInputStream extends ObjectInputStream {
/**
* Primitive types, strings and other built-in types do not pass through resolveClass() but
* instead have a custom encoding; see
* <a href="https://docs.oracle.com/javase/6/docs/platform/serialization/spec/protocol.html#10152">
* official docs</a>.
*/
private static final Set<String> CLASS_WHITELIST = Set.of(
"java.lang.Boolean",
"java.lang.Byte",
"java.lang.Character",
"java.lang.Short",
"java.lang.Integer",
"java.lang.Long",
"java.lang.Float",
"java.lang.Double",
"java.lang.Void",
"java.util.HashMap",
"java.util.HashSet"
);
public PreferencesObjectInputStream(final InputStream in) throws IOException {
super(in);
}
@Override
protected Class<?> resolveClass(final ObjectStreamClass desc)
throws ClassNotFoundException, IOException {
if (CLASS_WHITELIST.contains(desc.getName())) {
return super.resolveClass(desc);
} else {
throw new ClassNotFoundException("Class not allowed: " + desc.getName());
}
}
}

172
app/src/main/java/org/schabi/newpipe/util/ZipHelper.java
View File

@ -1,18 +1,21 @@
package org.schabi.newpipe.util;
import org.schabi.newpipe.streams.io.SharpInputStream;
import org.schabi.newpipe.streams.io.StoredFileHelper;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.schabi.newpipe.streams.io.StoredFileHelper;
/**
* Created by Christian Schabesberger on 28.01.18.
* Copyright 2018 Christian Schabesberger <chris.schabesberger@mailbox.org>
@ -34,73 +37,154 @@ import org.schabi.newpipe.streams.io.StoredFileHelper;
*/
public final class ZipHelper {
private ZipHelper() { }
private static final int BUFFER_SIZE = 2048;
@FunctionalInterface
public interface InputStreamConsumer {
void acceptStream(InputStream inputStream) throws IOException;
}
@FunctionalInterface
public interface OutputStreamConsumer {
void acceptStream(OutputStream outputStream) throws IOException;
}
private ZipHelper() { }
/**
* This function helps to create zip files.
* Caution this will override the original file.
* This function helps to create zip files. Caution this will overwrite the original file.
*
* @param outZip The ZipOutputStream where the data should be stored in
* @param file The path of the file that should be added to zip.
* @param name The path of the file inside the zip.
* @throws Exception
* @param outZip the ZipOutputStream where the data should be stored in
* @param nameInZip the path of the file inside the zip
* @param fileOnDisk the path of the file on the disk that should be added to zip
*/
public static void addFileToZip(final ZipOutputStream outZip, final String file,
final String name) throws Exception {
public static void addFileToZip(final ZipOutputStream outZip,
final String nameInZip,
final String fileOnDisk) throws IOException {
try (FileInputStream fi = new FileInputStream(fileOnDisk)) {
addFileToZip(outZip, nameInZip, fi);
}
}
/**
* This function helps to create zip files. Caution this will overwrite the original file.
*
* @param outZip the ZipOutputStream where the data should be stored in
* @param nameInZip the path of the file inside the zip
* @param streamConsumer will be called with an output stream that will go to the output file
*/
public static void addFileToZip(final ZipOutputStream outZip,
final String nameInZip,
final OutputStreamConsumer streamConsumer) throws IOException {
final byte[] bytes;
try (ByteArrayOutputStream byteOutput = new ByteArrayOutputStream()) {
streamConsumer.acceptStream(byteOutput);
bytes = byteOutput.toByteArray();
}
try (ByteArrayInputStream byteInput = new ByteArrayInputStream(bytes)) {
ZipHelper.addFileToZip(outZip, nameInZip, byteInput);
}
}
/**
* This function helps to create zip files. Caution this will overwrite the original file.
*
* @param outZip the ZipOutputStream where the data should be stored in
* @param nameInZip the path of the file inside the zip
* @param inputStream the content to put inside the file
*/
public static void addFileToZip(final ZipOutputStream outZip,
final String nameInZip,
final InputStream inputStream) throws IOException {
final byte[] data = new byte[BUFFER_SIZE];
try (FileInputStream fi = new FileInputStream(file);
BufferedInputStream inputStream = new BufferedInputStream(fi, BUFFER_SIZE)) {
final ZipEntry entry = new ZipEntry(name);
try (BufferedInputStream bufferedInputStream =
new BufferedInputStream(inputStream, BUFFER_SIZE)) {
final ZipEntry entry = new ZipEntry(nameInZip);
outZip.putNextEntry(entry);
int count;
while ((count = inputStream.read(data, 0, BUFFER_SIZE)) != -1) {
while ((count = bufferedInputStream.read(data, 0, BUFFER_SIZE)) != -1) {
outZip.write(data, 0, count);
}
}
}
/**
* This will extract data from ZipInputStream.
* Caution this will override the original file.
* This will extract data from ZipInputStream. Caution this will overwrite the original file.
*
* @param zipFile The zip file
* @param file The path of the file on the disk where the data should be extracted to.
* @param name The path of the file inside the zip.
* @param zipFile the zip file to extract from
* @param nameInZip the path of the file inside the zip
* @param fileOnDisk the path of the file on the disk where the data should be extracted to
* @return will return true if the file was found within the zip file
* @throws Exception
*/
public static boolean extractFileFromZip(final StoredFileHelper zipFile, final String file,
final String name) throws Exception {
public static boolean extractFileFromZip(final StoredFileHelper zipFile,
final String nameInZip,
final String fileOnDisk) throws IOException {
return extractFileFromZip(zipFile, nameInZip, input -> {
// delete old file first
final File oldFile = new File(fileOnDisk);
if (oldFile.exists()) {
if (!oldFile.delete()) {
throw new IOException("Could not delete " + fileOnDisk);
}
}
final byte[] data = new byte[BUFFER_SIZE];
try (FileOutputStream outFile = new FileOutputStream(fileOnDisk)) {
int count;
while ((count = input.read(data)) != -1) {
outFile.write(data, 0, count);
}
}
});
}
/**
* This will extract data from ZipInputStream.
*
* @param zipFile the zip file to extract from
* @param nameInZip the path of the file inside the zip
* @param streamConsumer will be called with the input stream from the file inside the zip
* @return will return true if the file was found within the zip file
*/
public static boolean extractFileFromZip(final StoredFileHelper zipFile,
final String nameInZip,
final InputStreamConsumer streamConsumer)
throws IOException {
try (ZipInputStream inZip = new ZipInputStream(new BufferedInputStream(
new SharpInputStream(zipFile.getStream())))) {
ZipEntry ze;
while ((ze = inZip.getNextEntry()) != null) {
if (ze.getName().equals(nameInZip)) {
streamConsumer.acceptStream(inZip);
return true;
}
}
return false;
}
}
/**
* @param zipFile the zip file
* @param fileInZip the filename to check
* @return whether the provided filename is in the zip; only the first level is checked
*/
public static boolean zipContainsFile(final StoredFileHelper zipFile, final String fileInZip)
throws Exception {
try (ZipInputStream inZip = new ZipInputStream(new BufferedInputStream(
new SharpInputStream(zipFile.getStream())))) {
final byte[] data = new byte[BUFFER_SIZE];
boolean found = false;
ZipEntry ze;
while ((ze = inZip.getNextEntry()) != null) {
if (ze.getName().equals(name)) {
found = true;
// delete old file first
final File oldFile = new File(file);
if (oldFile.exists()) {
if (!oldFile.delete()) {
throw new Exception("Could not delete " + file);
}
}
try (FileOutputStream outFile = new FileOutputStream(file)) {
int count = 0;
while ((count = inZip.read(data)) != -1) {
outFile.write(data, 0, count);
}
}
inZip.closeEntry();
if (ze.getName().equals(fileInZip)) {
return true;
}
}
return found;
return false;
}
}

1
app/src/main/res/values/strings.xml
View File

@ -856,4 +856,5 @@
</plurals>
<string name="show_more">Show more</string>
<string name="show_less">Show less</string>
<string name="import_settings_vulnerable_format">The settings in the export being imported use a vulnerable format that was deprecated since NewPipe 0.27.0. Make sure the export being imported is from a trusted source, and prefer using only exports obtained from NewPipe 0.27.0 or newer in the future. Support for importing settings in this vulnerable format will soon be removed completely, and then old versions of NewPipe will not be able to import settings of exports from new versions anymore.</string>
</resources>

184
app/src/test/java/org/schabi/newpipe/settings/ImportAllCombinationsTest.kt Normal file
View File

@ -0,0 +1,184 @@
package org.schabi.newpipe.settings
import android.content.SharedPreferences
import org.junit.Assert
import org.junit.Test
import org.mockito.Mockito
import org.schabi.newpipe.settings.export.BackupFileLocator
import org.schabi.newpipe.settings.export.ImportExportManager
import org.schabi.newpipe.streams.io.StoredFileHelper
import us.shandian.giga.io.FileStream
import java.io.File
import java.io.IOException
import java.nio.file.Files
class ImportAllCombinationsTest {
companion object {
private val classloader = ImportExportManager::class.java.classLoader!!
}
private enum class Ser(val id: String) {
YES("ser"),
VULNERABLE("vulnser"),
NO("noser");
}
private data class FailData(
val containsDb: Boolean,
val containsSer: Ser,
val containsJson: Boolean,
val filename: String,
val throwable: Throwable,
)
private fun testZipCombination(
containsDb: Boolean,
containsSer: Ser,
containsJson: Boolean,
filename: String,
runTest: (test: () -> Unit) -> Unit,
) {
val zipFile = File(classloader.getResource(filename)?.file!!)
val zip = Mockito.mock(StoredFileHelper::class.java, Mockito.withSettings().stubOnly())
Mockito.`when`(zip.stream).then { FileStream(zipFile) }
val fileLocator = Mockito.mock(
BackupFileLocator::class.java,
Mockito.withSettings().stubOnly()
)
val db = File.createTempFile("newpipe_", "")
val dbJournal = File.createTempFile("newpipe_", "")
val dbWal = File.createTempFile("newpipe_", "")
val dbShm = File.createTempFile("newpipe_", "")
Mockito.`when`(fileLocator.db).thenReturn(db)
Mockito.`when`(fileLocator.dbJournal).thenReturn(dbJournal)
Mockito.`when`(fileLocator.dbShm).thenReturn(dbShm)
Mockito.`when`(fileLocator.dbWal).thenReturn(dbWal)
if (containsDb) {
runTest {
Assert.assertTrue(ImportExportManager(fileLocator).extractDb(zip))
Assert.assertFalse(dbJournal.exists())
Assert.assertFalse(dbWal.exists())
Assert.assertFalse(dbShm.exists())
Assert.assertTrue("database file size is zero", Files.size(db.toPath()) > 0)
}
} else {
runTest {
Assert.assertFalse(ImportExportManager(fileLocator).extractDb(zip))
Assert.assertTrue(dbJournal.exists())
Assert.assertTrue(dbWal.exists())
Assert.assertTrue(dbShm.exists())
Assert.assertEquals(0, Files.size(db.toPath()))
}
}
val preferences = Mockito.mock(SharedPreferences::class.java, Mockito.withSettings().stubOnly())
var editor = Mockito.mock(SharedPreferences.Editor::class.java)
Mockito.`when`(preferences.edit()).thenReturn(editor)
Mockito.`when`(editor.commit()).thenReturn(true)
when (containsSer) {
Ser.YES -> runTest {
Assert.assertTrue(ImportExportManager(fileLocator).exportHasSerializedPrefs(zip))
ImportExportManager(fileLocator).loadSerializedPrefs(zip, preferences)
Mockito.verify(editor, Mockito.times(1)).clear()
Mockito.verify(editor, Mockito.times(1)).commit()
Mockito.verify(editor, Mockito.atLeastOnce())
.putBoolean(Mockito.anyString(), Mockito.anyBoolean())
Mockito.verify(editor, Mockito.atLeastOnce())
.putString(Mockito.anyString(), Mockito.anyString())
Mockito.verify(editor, Mockito.atLeastOnce())
.putInt(Mockito.anyString(), Mockito.anyInt())
}
Ser.VULNERABLE -> runTest {
Assert.assertTrue(ImportExportManager(fileLocator).exportHasSerializedPrefs(zip))
Assert.assertThrows(ClassNotFoundException::class.java) {
ImportExportManager(fileLocator).loadSerializedPrefs(zip, preferences)
}
Mockito.verify(editor, Mockito.never()).clear()
Mockito.verify(editor, Mockito.never()).commit()
}
Ser.NO -> runTest {
Assert.assertFalse(ImportExportManager(fileLocator).exportHasSerializedPrefs(zip))
Assert.assertThrows(IOException::class.java) {
ImportExportManager(fileLocator).loadSerializedPrefs(zip, preferences)
}
Mockito.verify(editor, Mockito.never()).clear()
Mockito.verify(editor, Mockito.never()).commit()
}
}
// recreate editor mock so verify() behaves correctly
editor = Mockito.mock(SharedPreferences.Editor::class.java)
Mockito.`when`(preferences.edit()).thenReturn(editor)
Mockito.`when`(editor.commit()).thenReturn(true)
if (containsJson) {
runTest {
Assert.assertTrue(ImportExportManager(fileLocator).exportHasJsonPrefs(zip))
ImportExportManager(fileLocator).loadJsonPrefs(zip, preferences)
Mockito.verify(editor, Mockito.times(1)).clear()
Mockito.verify(editor, Mockito.times(1)).commit()
Mockito.verify(editor, Mockito.atLeastOnce())
.putBoolean(Mockito.anyString(), Mockito.anyBoolean())
Mockito.verify(editor, Mockito.atLeastOnce())
.putString(Mockito.anyString(), Mockito.anyString())
Mockito.verify(editor, Mockito.atLeastOnce())
.putInt(Mockito.anyString(), Mockito.anyInt())
}
} else {
runTest {
Assert.assertFalse(ImportExportManager(fileLocator).exportHasJsonPrefs(zip))
Assert.assertThrows(IOException::class.java) {
ImportExportManager(fileLocator).loadJsonPrefs(zip, preferences)
}
Mockito.verify(editor, Mockito.never()).clear()
Mockito.verify(editor, Mockito.never()).commit()
}
}
}
@Test
fun `Importing all possible combinations of zip files`() {
val failedAssertions = mutableListOf<FailData>()
for (containsDb in listOf(true, false)) {
for (containsSer in Ser.entries) {
for (containsJson in listOf(true, false)) {
val filename = "settings/${if (containsDb) "db" else "nodb"}_${
containsSer.id}_${if (containsJson) "json" else "nojson"}.zip"
testZipCombination(containsDb, containsSer, containsJson, filename) { test ->
try {
test()
} catch (e: Throwable) {
failedAssertions.add(
FailData(
containsDb, containsSer, containsJson,
filename, e
)
)
}
}
}
}
}
if (failedAssertions.isNotEmpty()) {
for (a in failedAssertions) {
println(
"Assertion failed with containsDb=${a.containsDb}, containsSer=${
a.containsSer}, containsJson=${a.containsJson}, filename=${a.filename}:"
)
a.throwable.printStackTrace()
println()
}
Assert.fail("${failedAssertions.size} assertions failed")
}
}
}

92
app/src/test/java/org/schabi/newpipe/settings/ContentSettingsManagerTest.kt → app/src/test/java/org/schabi/newpipe/settings/ImportExportManagerTest.kt
View File

@ -1,8 +1,10 @@
package org.schabi.newpipe.settings
import android.content.SharedPreferences
import com.grack.nanojson.JsonParser
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertThrows
import org.junit.Assert.assertTrue
import org.junit.Assume
import org.junit.Before
@ -17,6 +19,8 @@ import org.mockito.Mockito.verify
import org.mockito.Mockito.`when`
import org.mockito.Mockito.withSettings
import org.mockito.junit.MockitoJUnitRunner
import org.schabi.newpipe.settings.export.BackupFileLocator
import org.schabi.newpipe.settings.export.ImportExportManager
import org.schabi.newpipe.streams.io.StoredFileHelper
import us.shandian.giga.io.FileStream
import java.io.File
@ -25,27 +29,25 @@ import java.nio.file.Files
import java.util.zip.ZipFile
@RunWith(MockitoJUnitRunner::class)
class ContentSettingsManagerTest {
class ImportExportManagerTest {
companion object {
private val classloader = ContentSettingsManager::class.java.classLoader!!
private val classloader = ImportExportManager::class.java.classLoader!!
}
private lateinit var fileLocator: NewPipeFileLocator
private lateinit var fileLocator: BackupFileLocator
private lateinit var storedFileHelper: StoredFileHelper
@Before
fun setupFileLocator() {
fileLocator = Mockito.mock(NewPipeFileLocator::class.java, withSettings().stubOnly())
fileLocator = Mockito.mock(BackupFileLocator::class.java, withSettings().stubOnly())
storedFileHelper = Mockito.mock(StoredFileHelper::class.java, withSettings().stubOnly())
}
@Test
fun `The settings must be exported successfully in the correct format`() {
val db = File(classloader.getResource("settings/newpipe.db")!!.file)
val newpipeSettings = File.createTempFile("newpipe_", "")
`when`(fileLocator.db).thenReturn(db)
`when`(fileLocator.settings).thenReturn(newpipeSettings)
val expectedPreferences = mapOf("such pref" to "much wow")
val sharedPreferences =
@ -54,11 +56,11 @@ class ContentSettingsManagerTest {
val output = File.createTempFile("newpipe_", "")
`when`(storedFileHelper.stream).thenReturn(FileStream(output))
ContentSettingsManager(fileLocator).exportDatabase(sharedPreferences, storedFileHelper)
ImportExportManager(fileLocator).exportDatabase(sharedPreferences, storedFileHelper)
val zipFile = ZipFile(output)
val entries = zipFile.entries().toList()
assertEquals(2, entries.size)
assertEquals(3, entries.size)
zipFile.getInputStream(entries.first { it.name == "newpipe.db" }).use { actual ->
db.inputStream().use { expected ->
@ -70,26 +72,11 @@ class ContentSettingsManagerTest {
val actualPreferences = ObjectInputStream(actual).readObject()
assertEquals(expectedPreferences, actualPreferences)
}
}
@Test
fun `Settings file must be deleted`() {
val settings = File.createTempFile("newpipe_", "")
`when`(fileLocator.settings).thenReturn(settings)
ContentSettingsManager(fileLocator).deleteSettingsFile()
assertFalse(settings.exists())
}
@Test
fun `Deleting settings file must do nothing if none exist`() {
val settings = File("non_existent")
`when`(fileLocator.settings).thenReturn(settings)
ContentSettingsManager(fileLocator).deleteSettingsFile()
assertFalse(settings.exists())
zipFile.getInputStream(entries.first { it.name == "preferences.json" }).use { actual ->
val actualPreferences = JsonParser.`object`().from(actual)
assertEquals(expectedPreferences, actualPreferences)
}
}
@Test
@ -98,7 +85,7 @@ class ContentSettingsManagerTest {
Assume.assumeTrue(dir.delete())
`when`(fileLocator.dbDir).thenReturn(dir)
ContentSettingsManager(fileLocator).ensureDbDirectoryExists()
ImportExportManager(fileLocator).ensureDbDirectoryExists()
assertTrue(dir.exists())
}
@ -107,7 +94,7 @@ class ContentSettingsManagerTest {
val dir = Files.createTempDirectory("newpipe_").toFile()
`when`(fileLocator.dbDir).thenReturn(dir)
ContentSettingsManager(fileLocator).ensureDbDirectoryExists()
ImportExportManager(fileLocator).ensureDbDirectoryExists()
assertTrue(dir.exists())
}
@ -122,9 +109,9 @@ class ContentSettingsManagerTest {
`when`(fileLocator.dbShm).thenReturn(dbShm)
`when`(fileLocator.dbWal).thenReturn(dbWal)
val zip = File(classloader.getResource("settings/newpipe.zip")?.file!!)
val zip = File(classloader.getResource("settings/db_ser_json.zip")?.file!!)
`when`(storedFileHelper.stream).thenReturn(FileStream(zip))
val success = ContentSettingsManager(fileLocator).extractDb(storedFileHelper)
val success = ImportExportManager(fileLocator).extractDb(storedFileHelper)
assertTrue(success)
assertFalse(dbJournal.exists())
@ -141,9 +128,9 @@ class ContentSettingsManagerTest {
val dbShm = File.createTempFile("newpipe_", "")
`when`(fileLocator.db).thenReturn(db)
val emptyZip = File(classloader.getResource("settings/empty.zip")?.file!!)
val emptyZip = File(classloader.getResource("settings/nodb_noser_nojson.zip")?.file!!)
`when`(storedFileHelper.stream).thenReturn(FileStream(emptyZip))
val success = ContentSettingsManager(fileLocator).extractDb(storedFileHelper)
val success = ImportExportManager(fileLocator).extractDb(storedFileHelper)
assertFalse(success)
assertTrue(dbJournal.exists())
@ -154,41 +141,44 @@ class ContentSettingsManagerTest {
@Test
fun `Contains setting must return true if a settings file exists in the zip`() {
val settings = File.createTempFile("newpipe_", "")
`when`(fileLocator.settings).thenReturn(settings)
val zip = File(classloader.getResource("settings/newpipe.zip")?.file!!)
val zip = File(classloader.getResource("settings/db_ser_json.zip")?.file!!)
`when`(storedFileHelper.stream).thenReturn(FileStream(zip))
val contains = ContentSettingsManager(fileLocator).extractSettings(storedFileHelper)
assertTrue(contains)
assertTrue(ImportExportManager(fileLocator).exportHasSerializedPrefs(storedFileHelper))
}
@Test
fun `Contains setting must return false if a no settings file exists in the zip`() {
val settings = File.createTempFile("newpipe_", "")
`when`(fileLocator.settings).thenReturn(settings)
val emptyZip = File(classloader.getResource("settings/empty.zip")?.file!!)
fun `Contains setting must return false if no settings file exists in the zip`() {
val emptyZip = File(classloader.getResource("settings/nodb_noser_nojson.zip")?.file!!)
`when`(storedFileHelper.stream).thenReturn(FileStream(emptyZip))
val contains = ContentSettingsManager(fileLocator).extractSettings(storedFileHelper)
assertFalse(contains)
assertFalse(ImportExportManager(fileLocator).exportHasSerializedPrefs(storedFileHelper))
}
@Test
fun `Preferences must be set from the settings file`() {
val settings = File(classloader.getResource("settings/newpipe.settings")!!.path)
`when`(fileLocator.settings).thenReturn(settings)
val zip = File(classloader.getResource("settings/db_ser_json.zip")?.file!!)
`when`(storedFileHelper.stream).thenReturn(FileStream(zip))
val preferences = Mockito.mock(SharedPreferences::class.java, withSettings().stubOnly())
val editor = Mockito.mock(SharedPreferences.Editor::class.java)
`when`(preferences.edit()).thenReturn(editor)
`when`(editor.commit()).thenReturn(true)
ContentSettingsManager(fileLocator).loadSharedPreferences(preferences)
ImportExportManager(fileLocator).loadSerializedPrefs(storedFileHelper, preferences)
verify(editor, atLeastOnce()).putBoolean(anyString(), anyBoolean())
verify(editor, atLeastOnce()).putString(anyString(), anyString())
verify(editor, atLeastOnce()).putInt(anyString(), anyInt())
}
@Test
fun `Importing preferences with a serialization injected class should fail`() {
val emptyZip = File(classloader.getResource("settings/db_vulnser_json.zip")?.file!!)
`when`(storedFileHelper.stream).thenReturn(FileStream(emptyZip))
val preferences = Mockito.mock(SharedPreferences::class.java, withSettings().stubOnly())
assertThrows(ClassNotFoundException::class.java) {
ImportExportManager(fileLocator).loadSerializedPrefs(storedFileHelper, preferences)
}
}
}

4
app/src/test/resources/settings/README.md Normal file
View File

@ -0,0 +1,4 @@
`*.zip` files in this folder are NewPipe database exports, in all possible configurations:
- `db` / `nodb` indicates if there is a `newpipe.db` database included or not
- `ser` / `vulnser` / `noser` indicates if there is a `newpipe.settings` Java-serialized preferences file included, if it is included and contains an injection attack, of if it is not included
- `json` / `nojson` indicates if there is a `preferences.json` JSON preferences file included or not

BIN
app/src/test/resources/settings/db_noser_json.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/db_noser_nojson.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/db_ser_json.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/db_ser_nojson.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/db_vulnser_json.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/db_vulnser_nojson.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/newpipe.settings
View File

Binary file not shown.

BIN
app/src/test/resources/settings/newpipe.zip
View File

Binary file not shown.

BIN
app/src/test/resources/settings/nodb_noser_json.zip Normal file
View File

Binary file not shown.

0
app/src/test/resources/settings/empty.zip → app/src/test/resources/settings/nodb_noser_nojson.zip
View File

BIN
app/src/test/resources/settings/nodb_ser_json.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/nodb_ser_nojson.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/nodb_vulnser_json.zip Normal file
View File

Binary file not shown.

BIN
app/src/test/resources/settings/nodb_vulnser_nojson.zip Normal file
View File

Binary file not shown.

4
checkstyle/checkstyle.xml
View File

@ -39,11 +39,13 @@
<module name="Translation"/>
<!-- Checks for Size Violations. -->
<!-- See https://checkstyle.org/config_sizes.html -->
<!-- See https://checkstyle.sourceforge.io/checks/sizes/index.html -->
<module name="FileLength"/>
<module name="LineLength">
<property name="max" value="100"/>
<property name="fileExtensions" value="java"/>
<!-- Also allow links in javadocs to be longer (the default would just cover imports) -->
<property name="ignorePattern" value="^((package|import) .*)|( *\* &lt;a href ?\= ?&quot;.*&quot;&gt;)$"/>
</module>
<!-- Checks for whitespace -->