From ab1550e2a3c30edfc14f2fe1e505619a7f612191 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:24:47 +0200 Subject: [PATCH 001/255] amd64: Save a copy of GENERIC as ELECTRO_BLOAT Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BLOAT | 365 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 sys/amd64/conf/ELECTRO_BLOAT diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT new file mode 100644 index 000000000000..3d2cc39c6353 --- /dev/null +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -0,0 +1,365 @@ +# +# GENERIC -- Generic kernel configuration file for FreeBSD/amd64 +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +cpu HAMMER +ident GENERIC + +makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols +makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support + +options SCHED_ULE # ULE scheduler +options PREEMPTION # Enable kernel thread preemption +options INET # InterNETworking +options INET6 # IPv6 communications protocols +options TCP_OFFLOAD # TCP offload +options SCTP # Stream Control Transmission Protocol +options FFS # Berkeley Fast Filesystem +options SOFTUPDATES # Enable FFS soft updates support +options UFS_ACL # Support for access control lists +options UFS_DIRHASH # Improve performance on big directories +options UFS_GJOURNAL # Enable gjournal-based UFS journaling +options QUOTA # Enable disk quotas for UFS +options MD_ROOT # MD is a potential root device +options NFSCL # Network Filesystem Client +options NFSD # Network Filesystem Server +options NFSLOCKD # Network Lock Manager +options NFS_ROOT # NFS usable as /, requires NFSCL +options MSDOSFS # MSDOS Filesystem +options CD9660 # ISO 9660 Filesystem +options PROCFS # Process filesystem (requires PSEUDOFS) +options PSEUDOFS # Pseudo-filesystem framework +options GEOM_PART_GPT # GUID Partition Tables. +options GEOM_RAID # Soft RAID functionality. +options GEOM_LABEL # Provides labelization +options COMPAT_FREEBSD32 # Compatible with i386 binaries +options COMPAT_FREEBSD4 # Compatible with FreeBSD4 +options COMPAT_FREEBSD5 # Compatible with FreeBSD5 +options COMPAT_FREEBSD6 # Compatible with FreeBSD6 +options COMPAT_FREEBSD7 # Compatible with FreeBSD7 +options COMPAT_FREEBSD9 # Compatible with FreeBSD9 +options COMPAT_FREEBSD10 # Compatible with FreeBSD10 +options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI +options KTRACE # ktrace(1) support +options STACK # stack(9) support +options SYSVSHM # SYSV-style shared memory +options SYSVMSG # SYSV-style message queues +options SYSVSEM # SYSV-style semaphores +options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions +options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. +options KBD_INSTALL_CDEV # install a CDEV entry in /dev +options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) +options AUDIT # Security event auditing +options CAPABILITY_MODE # Capsicum capability mode +options CAPABILITIES # Capsicum capabilities +options MAC # TrustedBSD MAC Framework +options KDTRACE_FRAME # Ensure frames are compiled in +options KDTRACE_HOOKS # Kernel DTrace hooks +options DDB_CTF # Kernel ELF linker loads CTF data +options INCLUDE_CONFIG_FILE # Include this file in kernel +options RACCT # Resource accounting framework +options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default +options RCTL # Resource limits + +# Debugging support. Always need this: +options KDB # Enable kernel debugger support. +options KDB_TRACE # Print a stack trace for a panic. +# For full debugger support use (turn off in stable branch): +options DDB # Support DDB. +options GDB # Support remote GDB. +options DEADLKRES # Enable the deadlock resolver +options INVARIANTS # Enable calls of extra sanity checking +options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +options WITNESS # Enable checks to detect deadlocks and cycles +options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones + +# Make an SMP-capable kernel by default +options SMP # Symmetric MultiProcessor Kernel + +# CPU frequency control +device cpufreq + +# Bus support. +device acpi +options ACPI_DMAR +device pci +options PCI_IOV # PCI SR-IOV support + +# Floppy drives +device fdc + +# ATA controllers +device ahci # AHCI-compatible SATA controllers +device ata # Legacy ATA/SATA controllers +options ATA_STATIC_ID # Static device numbering +device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +device ahc # AHA2940 and onboard AIC7xxx devices +options AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +device ahd # AHA39320/29320 and onboard AIC79xx devices +options AHD_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~215k to driver. +device esp # AMD Am53C974 (Tekram DC-390(T)) +device hptiop # Highpoint RocketRaid 3xxx series +device isp # Qlogic family +#device ispfw # Firmware for QLogic HBAs- normally a module +device mpt # LSI-Logic MPT-Fusion +device mps # LSI-Logic MPT-Fusion 2 +device mpr # LSI-Logic MPT-Fusion 3 +#device ncr # NCR/Symbios Logic +device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +device trm # Tekram DC395U/UW/F DC315U adapters + +device adv # Advansys SCSI adapters +device adw # Advansys wide SCSI adapters +device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +device bt # Buslogic/Mylex MultiMaster SCSI adapters +device isci # Intel C600 SAS controller + +# ATA/SCSI peripherals +device scbus # SCSI bus (required for ATA/SCSI) +device ch # SCSI media changers +device da # Direct Access (disks) +device sa # Sequential Access (tape etc) +device cd # CD +device pass # Passthrough device (direct ATA/SCSI access) +device ses # Enclosure Services (SES and SAF-TE) +#device ctl # CAM Target Layer + +# RAID controllers interfaced to the SCSI subsystem +device amr # AMI MegaRAID +device arcmsr # Areca SATA II RAID +device ciss # Compaq Smart RAID 5* +device dpt # DPT Smartcache III, IV - See NOTES for options +device hptmv # Highpoint RocketRAID 182x +device hptnr # Highpoint DC7280, R750 +device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +device hpt27xx # Highpoint RocketRAID 27xx +device iir # Intel Integrated RAID +device ips # IBM (Adaptec) ServeRAID +device mly # Mylex AcceleRAID/eXtremeRAID +device twa # 3ware 9000 series PATA/SATA RAID +device tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +device aac # Adaptec FSA RAID +device aacp # SCSI passthrough for aac (requires CAM) +device aacraid # Adaptec by PMC RAID +device ida # Compaq Smart RAID +device mfi # LSI MegaRAID SAS +device mlx # Mylex DAC960 family +device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s +#XXX pointer/int warnings +#device pst # Promise Supertrak SX6000 +device twe # 3ware ATA RAID + +# NVM Express (NVMe) support +device nvme # base NVMe driver +device nvd # expose NVMe namespaces as disks, depends on nvme + +# atkbdc0 controls both the keyboard and the PS/2 mouse +device atkbdc # AT keyboard controller +device atkbd # AT keyboard +device psm # PS/2 mouse + +device kbdmux # keyboard multiplexer + +device vga # VGA video card driver +options VESA # Add support for VESA BIOS Extensions (VBE) + +device splash # Splash screen and screen saver support + +# syscons is the default console driver, resembling an SCO console +device sc +options SC_PIXEL_MODE # add support for the raster text mode + +# vt is the new video console driver +device vt +device vt_vga +device vt_efifb + +device agp # support several AGP chipsets + +# PCCARD (PCMCIA) support +# PCMCIA and cardbus bridge support +device cbb # cardbus (yenta) bridge +device pccard # PC Card (16-bit) bus +device cardbus # CardBus (32-bit) bus + +# Serial (COM) ports +device uart # Generic UART driver + +# Parallel port +device ppc +device ppbus # Parallel port bus (required) +device lpt # Printer +device ppi # Parallel port interface device +#device vpo # Requires scbus and da + +device puc # Multi I/O cards and multi-channel UARTs + +# PCI Ethernet NICs. +device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +device de # DEC/Intel DC21x4x (``Tulip'') +device em # Intel PRO/1000 Gigabit Ethernet Family +device igb # Intel PRO/1000 PCIE Server Gigabit Family +device ix # Intel PRO/10GbE PCIE PF Ethernet +device ixv # Intel PRO/10GbE PCIE VF Ethernet +device ixl # Intel XL710 40Gbe PCIE Ethernet +device ixlv # Intel XL710 40Gbe VF PCIE Ethernet +device le # AMD Am7900 LANCE and Am79C9xx PCnet +device ti # Alteon Networks Tigon I/II gigabit Ethernet +device txp # 3Com 3cR990 (``Typhoon'') +device vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! +device miibus # MII bus support +device ae # Attansic/Atheros L2 FastEthernet +device age # Attansic/Atheros L1 Gigabit Ethernet +device alc # Atheros AR8131/AR8132 Ethernet +device ale # Atheros AR8121/AR8113/AR8114 Ethernet +device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +device bfe # Broadcom BCM440x 10/100 Ethernet +device bge # Broadcom BCM570xx Gigabit Ethernet +device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +device dc # DEC/Intel 21143 and various workalikes +device et # Agere ET1310 10/100/Gigabit Ethernet +device fxp # Intel EtherExpress PRO/100B (82557, 82558) +device gem # Sun GEM/Sun ERI/Apple GMAC +device hme # Sun HME (Happy Meal Ethernet) +device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +device lge # Level 1 LXT1001 gigabit Ethernet +device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +device nfe # nVidia nForce MCP on-board Ethernet +device nge # NatSemi DP83820 gigabit Ethernet +device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +device re # RealTek 8139C+/8169/8169S/8110S +device rl # RealTek 8129/8139 +device sf # Adaptec AIC-6915 (``Starfire'') +device sge # Silicon Integrated Systems SiS190/191 +device sis # Silicon Integrated Systems SiS 900/SiS 7016 +device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +device ste # Sundance ST201 (D-Link DFE-550TX) +device stge # Sundance/Tamarack TC9021 gigabit Ethernet +device tl # Texas Instruments ThunderLAN +device tx # SMC EtherPower II (83c170 ``EPIC'') +device vge # VIA VT612x gigabit Ethernet +device vr # VIA Rhine, Rhine II +device wb # Winbond W89C840F +device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# Wireless NIC cards +device wlan # 802.11 support +options IEEE80211_DEBUG # enable debug msgs +options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's +options IEEE80211_SUPPORT_MESH # enable 802.11s draft support +device wlan_wep # 802.11 WEP support +device wlan_ccmp # 802.11 CCMP support +device wlan_tkip # 802.11 TKIP support +device wlan_amrr # AMRR transmit rate control algorithm +device an # Aironet 4500/4800 802.11 wireless NICs. +device ath # Atheros NICs +device ath_pci # Atheros pci/cardbus glue +device ath_hal # pci/cardbus chip support +options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors +options AH_AR5416_INTERRUPT_MITIGATION # AR5416 interrupt mitigation +options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later +device ath_rate_sample # SampleRate tx rate control for ath +#device bwi # Broadcom BCM430x/BCM431x wireless NICs. +#device bwn # Broadcom BCM43xx wireless NICs. +device ipw # Intel 2100 wireless NICs. +device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +device iwn # Intel 4965/1000/5000/6000 wireless NICs. +device malo # Marvell Libertas wireless NICs. +device mwl # Marvell 88W8363 802.11n wireless NICs. +device ral # Ralink Technology RT2500 wireless NICs. +device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +device wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +device loop # Network loopback +device random # Entropy device +device padlock_rng # VIA Padlock RNG +device rdrand_rng # Intel Bull Mountain RNG +device ether # Ethernet support +device vlan # 802.1Q VLAN support +device tun # Packet tunnel. +device md # Memory "disks" +device gif # IPv6 and IPv4 tunneling +device firmware # firmware assist module + +# The `bpf' device enables the Berkeley Packet Filter. +# Be aware of the administrative consequences of enabling this! +# Note that 'bpf' is required for DHCP. +device bpf # Berkeley packet filter + +# USB support +options USB_DEBUG # enable debug msgs +device uhci # UHCI PCI->USB interface +device ohci # OHCI PCI->USB interface +device ehci # EHCI PCI->USB interface (USB 2.0) +device xhci # XHCI PCI->USB interface (USB 3.0) +device usb # USB Bus (required) +device ukbd # Keyboard +device umass # Disks/Mass storage - Requires scbus and da + +# Sound support +device sound # Generic sound driver (required) +device snd_cmi # CMedia CMI8338/CMI8738 +device snd_csa # Crystal Semiconductor CS461x/428x +device snd_emu10kx # Creative SoundBlaster Live! and Audigy +device snd_es137x # Ensoniq AudioPCI ES137x +device snd_hda # Intel High Definition Audio +device snd_ich # Intel, NVidia and other ICH AC'97 Audio +device snd_via8233 # VIA VT8233x Audio + +# MMC/SD +device mmc # MMC/SD bus +device mmcsd # MMC/SD memory card +device sdhci # Generic PCI SD Host Controller + +# VirtIO support +device virtio # Generic VirtIO bus (required) +device virtio_pci # VirtIO PCI device +device vtnet # VirtIO Ethernet device +device virtio_blk # VirtIO Block device +device virtio_scsi # VirtIO SCSI device +device virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers and enchancement support +# NOTE: HYPERV depends on hyperv. They must be added or removed together. +options HYPERV # Hyper-V kernel infrastructure +device hyperv # HyperV drivers + +# Xen HVM Guest Optimizations +# NOTE: XENHVM depends on xenpci. They must be added or removed together. +options XENHVM # Xen HVM kernel infrastructure +device xenpci # Xen HVM Hypervisor services driver + +# VMware support +device vmx # VMware VMXNET3 Ethernet + +# Netmap provides direct access to TX/RX rings on supported NICs +device netmap # netmap(4) support + -- 2.11.0 From fbcf856c0599975f001a9c443c9520dfb6857d90 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:30:20 +0200 Subject: [PATCH 002/255] ELECTRO_BLOAT: ElectroBSDify the copy Mainly by removing stuff that depends on proprietary software ElectroBSD doesn't ship with or stuff that isn't relevant for ElectroBSD. While at it, import the 'device crypto' line from GENERIC. While we don't need it for IPSEC, having it in the kernel makes upgrading from FreeBSD to ElectroBSD more convenient when using cloudiatr. By default cloudiatr only adds currenty-loaded modules to the bpool, so if crypto.ko is part of the currently running kernel, but the installed kernel requires it as module, the newly-setup system will not boot unless the user adds the module manually. Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BLOAT | 76 +++++++++++++------------------------------- 1 file changed, 22 insertions(+), 54 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index 3d2cc39c6353..a61f4e5eee3a 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -1,25 +1,25 @@ -# -# GENERIC -- Generic kernel configuration file for FreeBSD/amd64 -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ +# ELECTRO_BLOAT -- Modified copy of the GENERIC kernel configuration file +# Used for the release media. +ident ELECTRO_BLOAT + +# One of ElectroBSD's most important features: +# hacker-movie-compatible colors by default! +options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) + +# Add HTTP accept filter support. The "performance gains" might +# be dubious, but adding it results in nicer logs for applications +# that use it (because requests are less intangled). +options ACCEPT_FILTER_HTTP + +# Build pf into the kernel. It doesn't hurt and supposedly +# works around various bugs that only affect the module build. +device pf + +############################################################################## +# Everything below comes from GENERIC, but "offending" lines have been removed +############################################################################## cpu HAMMER -ident GENERIC makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support @@ -29,7 +29,6 @@ options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols options TCP_OFFLOAD # TCP offload -options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists @@ -84,10 +83,6 @@ options KDB_TRACE # Print a stack trace for a panic. options DDB # Support DDB. options GDB # Support remote GDB. options DEADLKRES # Enable the deadlock resolver -options INVARIANTS # Enable calls of extra sanity checking -options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS -options WITNESS # Enable checks to detect deadlocks and cycles -options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # Make an SMP-capable kernel by default @@ -102,9 +97,6 @@ options ACPI_DMAR device pci options PCI_IOV # PCI SR-IOV support -# Floppy drives -device fdc - # ATA controllers device ahci # AHCI-compatible SATA controllers device ata # Legacy ATA/SATA controllers @@ -121,13 +113,9 @@ options AHD_REG_PRETTY_PRINT # Print register bitfields in debug # output. Adds ~215k to driver. device esp # AMD Am53C974 (Tekram DC-390(T)) device hptiop # Highpoint RocketRaid 3xxx series -device isp # Qlogic family -#device ispfw # Firmware for QLogic HBAs- normally a module device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -#device ncr # NCR/Symbios Logic -device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters device adv # Advansys SCSI adapters @@ -151,10 +139,6 @@ device amr # AMI MegaRAID device arcmsr # Areca SATA II RAID device ciss # Compaq Smart RAID 5* device dpt # DPT Smartcache III, IV - See NOTES for options -device hptmv # Highpoint RocketRAID 182x -device hptnr # Highpoint DC7280, R750 -device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx -device hpt27xx # Highpoint RocketRAID 27xx device iir # Intel Integrated RAID device ips # IBM (Adaptec) ServeRAID device mly # Mylex AcceleRAID/eXtremeRAID @@ -167,10 +151,7 @@ device aacp # SCSI passthrough for aac (requires CAM) device aacraid # Adaptec by PMC RAID device ida # Compaq Smart RAID device mfi # LSI MegaRAID SAS -device mlx # Mylex DAC960 family device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s -#XXX pointer/int warnings -#device pst # Promise Supertrak SX6000 device twe # 3ware ATA RAID # NVM Express (NVMe) support @@ -219,7 +200,6 @@ device ppi # Parallel port interface device device puc # Multi I/O cards and multi-channel UARTs # PCI Ethernet NICs. -device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family device igb # Intel PRO/1000 PCIE Server Gigabit Family @@ -239,7 +219,6 @@ device ae # Attansic/Atheros L2 FastEthernet device age # Attansic/Atheros L1 Gigabit Ethernet device alc # Atheros AR8131/AR8132 Ethernet device ale # Atheros AR8121/AR8113/AR8114 Ethernet -device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet device bfe # Broadcom BCM440x 10/100 Ethernet device bge # Broadcom BCM570xx Gigabit Ethernet device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn @@ -288,14 +267,7 @@ options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later device ath_rate_sample # SampleRate tx rate control for ath #device bwi # Broadcom BCM430x/BCM431x wireless NICs. #device bwn # Broadcom BCM43xx wireless NICs. -device ipw # Intel 2100 wireless NICs. -device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. -device iwn # Intel 4965/1000/5000/6000 wireless NICs. device malo # Marvell Libertas wireless NICs. -device mwl # Marvell 88W8363 802.11n wireless NICs. -device ral # Ralink Technology RT2500 wireless NICs. -device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. -device wpi # Intel 3945ABG wireless NICs. # Pseudo devices. device loop # Network loopback @@ -327,7 +299,6 @@ device umass # Disks/Mass storage - Requires scbus and da # Sound support device sound # Generic sound driver (required) device snd_cmi # CMedia CMI8338/CMI8738 -device snd_csa # Crystal Semiconductor CS461x/428x device snd_emu10kx # Creative SoundBlaster Live! and Audigy device snd_es137x # Ensoniq AudioPCI ES137x device snd_hda # Intel High Definition Audio @@ -348,8 +319,6 @@ device virtio_scsi # VirtIO SCSI device device virtio_balloon # VirtIO Memory Balloon device # HyperV drivers and enchancement support -# NOTE: HYPERV depends on hyperv. They must be added or removed together. -options HYPERV # Hyper-V kernel infrastructure device hyperv # HyperV drivers # Xen HVM Guest Optimizations @@ -360,6 +329,5 @@ device xenpci # Xen HVM Hypervisor services driver # VMware support device vmx # VMware VMXNET3 Ethernet -# Netmap provides direct access to TX/RX rings on supported NICs -device netmap # netmap(4) support - +# The crypto framework is required by IPSEC +device crypto # Required by IPSEC -- 2.11.0 From aa94490bde211cb8222a7272d14665baa9feaaa7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Mar 2013 11:28:27 +0100 Subject: [PATCH 003/255] Import ELECTRO_BEER Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BEER | 257 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 sys/amd64/conf/ELECTRO_BEER diff --git a/sys/amd64/conf/ELECTRO_BEER b/sys/amd64/conf/ELECTRO_BEER new file mode 100644 index 000000000000..3fb26ab1675f --- /dev/null +++ b/sys/amd64/conf/ELECTRO_BEER @@ -0,0 +1,257 @@ +include ELECTRO_BLOAT + +ident ELECTRO_BEER + +# Debugging for use in -current +nooptions INVARIANTS # Enable calls of extra sanity checking +nooptions INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +nooptions WITNESS # Enable checks to detect deadlocks and cycles +nooptions WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +nooptions FLOWTABLE # per-cpu routing cache +nooptions XENHVM # Include Xen support + +nooptions SCTP # Stream Control Transmission Protocol + +nodevice fdc + +nodevice ataraid # ATA RAID drives +nodevice atapifd # ATAPI floppy drives +nodevice atapist # ATAPI tape drives + +# ATA controllers +nodevice mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +nodevice siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +nodevice ahc # AHA2940 and onboard AIC7xxx devices +nooptions AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +nodevice amd # AMD 53C974 (Tekram DC-390(T)) +nodevice esp # AMD Am53C974 (Tekram DC-390(T)) +nodevice hptiop # Highpoint RocketRaid 3xxx series +nodevice isp # Qlogic family +nodevice ispfw # Firmware for QLogic HBAs- normally a module +nodevice mpt # LSI-Logic MPT-Fusion +nodevice mps # LSI-Logic MPT-Fusion 2 +nodevice ncr # NCR/Symbios Logic +nodevice sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +nodevice trm # Tekram DC395U/UW/F DC315U adapters + +nodevice adv # Advansys SCSI adapters +nodevice adw # Advansys wide SCSI adapters +nodevice aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +nodevice bt # Buslogic/Mylex MultiMaster SCSI adapters + +# RAID controllers interfaced to the SCSI subsystem +nodevice amr # AMI MegaRAID +nodevice arcmsr # Areca SATA II RAID +#XXX it is not 64-bit clean, -scottl +nodevice asr # DPT SmartRAID V, VI and Adaptec SCSI RAID +nodevice ciss # Compaq Smart RAID 5* +nodevice dpt # DPT Smartcache III, IV - See NOTES for options +nodevice hptmv # Highpoint RocketRAID 182x +nodevice hptnr # Highpoint DC7280, R750 +nodevice hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +nodevice hpt27xx # Highpoint RocketRAID 27xx + +nodevice iir # Intel Integrated RAID +nodevice ips # IBM (Adaptec) ServeRAID +nodevice mly # Mylex AcceleRAID/eXtremeRAID +nodevice twa # 3ware 9000 series PATA/SATA RAID +nodevice tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +nodevice aac # Adaptec FSA RAID +nodevice aacraid # Adaptec by PMC RAID +nodevice aacp # SCSI passthrough for aac (requires CAM) +nodevice ida # Compaq Smart RAID +nodevice mfi # LSI MegaRAID SAS +nodevice mlx # Mylex DAC960 family +#XXX pointer/int warnings +nodevice pst # Promise Supertrak SX6000 +nodevice twe # 3ware ATA RAID + + +# Parallel port +nodevice ppc +nodevice ppbus # Parallel port bus (required) +nodevice lpt # Printer +nodevice plip # TCP/IP over parallel +nodevice ppi # Parallel port interface device +nodevice vpo # Requires scbus and da + +# If you've got a "dumb" serial or parallel PCI card that is +# supported by the puc(4) glue driver, uncomment the following +# line to enable it (connects to sio, uart and/or ppc drivers): +nodevice puc + +# PCI Ethernet NICs. +nodevice bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +nodevice de # DEC/Intel DC21x4x (``Tulip'') +nodevice em # Intel PRO/1000 Gigabit Ethernet Family +nodevice igb # Intel PRO/1000 PCIE Server Gigabit Family +nodevice ixgbe # Intel PRO/10GbE PCIE Ethernet Family +nodevice le # AMD Am7900 LANCE and Am79C9xx PCnet +nodevice ti # Alteon Networks Tigon I/II gigabit Ethernet +nodevice txp # 3Com 3cR990 (``Typhoon'') +nodevice vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! + +nodevice ae # Attansic/Atheros L2 FastEthernet +nodevice age # Attansic/Atheros L1 Gigabit Ethernet +nodevice alc # Atheros AR8131/AR8132 Ethernet +nodevice ale # Atheros AR8121/AR8113/AR8114 Ethernet +nodevice bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +nodevice bfe # Broadcom BCM440x 10/100 Ethernet +nodevice cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +nodevice dc # DEC/Intel 21143 and various workalikes +nodevice et # Agere ET1310 10/100/Gigabit Ethernet +nodevice fxp # Intel EtherExpress PRO/100B (82557, 82558) +nodevice gem # Sun GEM/Sun ERI/Apple GMAC +nodevice hme # Sun HME (Happy Meal Ethernet) +nodevice jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +nodevice lge # Level 1 LXT1001 gigabit Ethernet +nodevice msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +nodevice nfe # nVidia nForce MCP on-board Ethernet +nodevice nge # NatSemi DP83820 gigabit Ethernet +nodevice nve # nVidia nForce MCP on-board Ethernet Networking +nodevice pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +nodevice re # RealTek 8139C+/8169/8169S/8110S +nodevice rl # RealTek 8129/8139 +nodevice sf # Adaptec AIC-6915 (``Starfire'') +nodevice sge # Silicon Integrated Systems SiS190/191 +nodevice sis # Silicon Integrated Systems SiS 900/SiS 7016 +nodevice sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +nodevice ste # Sundance ST201 (D-Link DFE-550TX) +nodevice stge # Sundance/Tamarack TC9021 gigabit Ethernet +nodevice tl # Texas Instruments ThunderLAN +nodevice tx # SMC EtherPower II (83c170 ``EPIC'') +nodevice vge # VIA VT612x gigabit Ethernet +nodevice vr # VIA Rhine, Rhine II +nodevice wb # Winbond W89C840F +nodevice xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# ISA Ethernet NICs. pccard NICs included. +nodevice cs # Crystal Semiconductor CS89x0 NIC +# 'device ed' requires 'device miibus' +nodevice ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards +nodevice ex # Intel EtherExpress Pro/10 and Pro/10+ +nodevice ep # Etherlink III based cards +nodevice fe # Fujitsu MB8696x based cards +nodevice sn # SMC's 9000 series of Ethernet chips +nodevice xe # Xircom pccard Ethernet + +nodevice zyd # ZyDAS zb1211/zb1211b wireless NICs +nodevice urio # Diamond Rio 500 MP3 player + +# Wireless NIC cards +nodevice an # Aironet 4500/4800 802.11 wireless NICs. +nodevice ath # Atheros NIC's +nodevice ath_pci # Atheros pci/cardbus glue +nodevice ath_hal # pci/cardbus chip support +nodevice ath_rate_sample # SampleRate tx rate control for ath +nodevice bwi # Broadcom BCM430x/BCM431x wireless NICs. +nodevice bwn # Broadcom BCM43xx wireless NICs. +nodevice ipw # Intel 2100 wireless NICs. +nodevice iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +nodevice iwn # Intel 4965/1000/5000/6000 wireless NICs. +nodevice malo # Marvell Libertas wireless NICs. +nodevice mwl # Marvell 88W8363 802.11n wireless NICs. +nodevice ral # Ralink Technology RT2500 wireless NICs. +nodevice wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +nodevice wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +nodevice padlock_rng # VIA Padlock RNG +nodevice rdrand_rng # Intel Bull Mountain RNG +nodevice gif # IPv6 and IPv4 tunneling +nodevice faith # IPv6-to-IPv4 relaying (translation) + +# USB Serial devices +nodevice uark # Technologies ARK3116 based serial adapters +nodevice ubsa # Belkin F5U103 and compatible serial adapters +nodevice uftdi # For FTDI usb serial adapters +nodevice uipaq # Some WinCE based devices +nodevice uplcom # Prolific PL-2303 serial adapters +nodevice uslcom # SI Labs CP2101/CP2102 serial adapters +nodevice uvisor # Visor and Palm devices +nodevice uvscom # USB serial support for DDI pocket's PHS +# USB Ethernet, requires miibus +nodevice aue # ADMtek USB Ethernet +nodevice axe # ASIX Electronics USB Ethernet +nodevice cdce # Generic USB over Ethernet +nodevice cue # CATC USB Ethernet +nodevice kue # Kawasaki LSI USB Ethernet +nodevice rue # RealTek RTL8150 USB Ethernet +nodevice udav # Davicom DM9601E USB + +# USB support + +nodevice uhci # UHCI PCI->USB interface +nodevice ohci # OHCI PCI->USB interface +nodevice ehci # EHCI PCI->USB interface (USB 2.0) +nodevice xhci # XHCI PCI->USB interface (USB 3.0) +nodevice usb # USB Bus (required) +nodevice ukbd # Keyboard +nodevice umass # Disks/Mass storage - Requires scbus and da + +# FireWire support +nodevice firewire # FireWire bus code +nodevice sbp # SCSI over FireWire (Requires scbus and da) +nodevice fwe # Ethernet over FireWire (non-standard!) +nodevice fwip # IP over FireWire (RFC 2734,3146) +nodevice dcons # Dumb console driver +nodevice dcons_crom # Configuration ROM for dcons + +# Sound support +nodevice snd_es137x # Ensoniq AudioPCI ES137x +nodevice snd_ich # Intel, NVidia and other ICH AC'97 Audio +nodevice snd_uaudio # USB Audio +nodevice snd_via8233 # VIA VT8233x Audio +nodevice snd_cmi # CMedia CMI8338/CMI8738 +nodevice snd_csa # Crystal Semiconductor CS461x/428x +nodevice snd_emu10kx # Creative SoundBlaster Live! and Audigy + +# VirtIO support +nodevice virtio # Generic VirtIO bus (required) +nodevice virtio_pci # VirtIO PCI device +nodevice vtnet # VirtIO Ethernet device +nodevice virtio_blk # VirtIO Block device +nodevice virtio_scsi # VirtIO SCSI device +nodevice virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers +nodevice hyperv # HyperV drivers + +# Xen support +nodevice xenpci # Generic Xen bus + +# VMware support +nodevice vmx # VMware VMXNET3 Ethernet + +# Same for Intel processors +device coretemp + +# man 4 cpuctl +device cpuctl # CPU control pseudo-device + +# UTF-8 in console (8.x+) +options TEKEN_UTF8 + +nodevice netmap + +# Disabling them prevents ZFS from being loaded. +# Should be bisected. +#nooptions NFS_ROOT +#nooptions NFSLOCKD +#nooptions NFSD +#nooptions NFSCL +nodevice ahd +nodevice mpr +nodevice isci +nodevice ses +nodevice mrsas +nodevice ixl +nodevice ixlv -- 2.11.0 From e6ec5a267575f79950f6d0889e653ee483a4b115 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 14 Apr 2015 17:43:38 +0200 Subject: [PATCH 004/255] Add ELECTRO_BEER for i386 Obtained from: ElectroBSD --- sys/i386/conf/ELECTRO_BEER | 1 + 1 file changed, 1 insertion(+) create mode 120000 sys/i386/conf/ELECTRO_BEER diff --git a/sys/i386/conf/ELECTRO_BEER b/sys/i386/conf/ELECTRO_BEER new file mode 120000 index 000000000000..4a483bb3a8ea --- /dev/null +++ b/sys/i386/conf/ELECTRO_BEER @@ -0,0 +1 @@ +../../amd64/conf/ELECTRO_BEER \ No newline at end of file -- 2.11.0 From 84a531cd16f8f52459ab951d497173a33cd9a04e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 17:06:35 +0200 Subject: [PATCH 005/255] i386: Copy GENERIC to ELECTRO_BLOAT Obtained from: ElectroBSD --- sys/i386/conf/ELECTRO_BLOAT | 382 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 sys/i386/conf/ELECTRO_BLOAT diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT new file mode 100644 index 000000000000..aefc50753f0d --- /dev/null +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -0,0 +1,382 @@ +# +# GENERIC -- Generic kernel configuration file for FreeBSD/i386 +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +cpu I486_CPU +cpu I586_CPU +cpu I686_CPU +ident GENERIC + +makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols +makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support + +options SCHED_ULE # ULE scheduler +options PREEMPTION # Enable kernel thread preemption +options INET # InterNETworking +options INET6 # IPv6 communications protocols +options IPSEC # IP (v4/v6) security +options TCP_OFFLOAD # TCP offload +options SCTP # Stream Control Transmission Protocol +options FFS # Berkeley Fast Filesystem +options SOFTUPDATES # Enable FFS soft updates support +options UFS_ACL # Support for access control lists +options UFS_DIRHASH # Improve performance on big directories +options UFS_GJOURNAL # Enable gjournal-based UFS journaling +options QUOTA # Enable disk quotas for UFS +options MD_ROOT # MD is a potential root device +options NFSCL # Network Filesystem Client +options NFSD # Network Filesystem Server +options NFSLOCKD # Network Lock Manager +options NFS_ROOT # NFS usable as /, requires NFSCL +options MSDOSFS # MSDOS Filesystem +options CD9660 # ISO 9660 Filesystem +options PROCFS # Process filesystem (requires PSEUDOFS) +options PSEUDOFS # Pseudo-filesystem framework +options GEOM_PART_GPT # GUID Partition Tables. +options GEOM_RAID # Soft RAID functionality. +options GEOM_LABEL # Provides labelization +options COMPAT_FREEBSD4 # Compatible with FreeBSD4 +options COMPAT_FREEBSD5 # Compatible with FreeBSD5 +options COMPAT_FREEBSD6 # Compatible with FreeBSD6 +options COMPAT_FREEBSD7 # Compatible with FreeBSD7 +options COMPAT_FREEBSD9 # Compatible with FreeBSD9 +options COMPAT_FREEBSD10 # Compatible with FreeBSD10 +options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI +options KTRACE # ktrace(1) support +options STACK # stack(9) support +options SYSVSHM # SYSV-style shared memory +options SYSVMSG # SYSV-style message queues +options SYSVSEM # SYSV-style semaphores +options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions +options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. +options KBD_INSTALL_CDEV # install a CDEV entry in /dev +options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) +options AUDIT # Security event auditing +options CAPABILITY_MODE # Capsicum capability mode +options CAPABILITIES # Capsicum capabilities +options MAC # TrustedBSD MAC Framework +options KDTRACE_HOOKS # Kernel DTrace hooks +options DDB_CTF # Kernel ELF linker loads CTF data +options INCLUDE_CONFIG_FILE # Include this file in kernel +options RACCT # Resource accounting framework +options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default +options RCTL # Resource limits + +# Debugging support. Always need this: +options KDB # Enable kernel debugger support. +options KDB_TRACE # Print a stack trace for a panic. +# For full debugger support use (turn off in stable branch): +options DDB # Support DDB. +options GDB # Support remote GDB. +options DEADLKRES # Enable the deadlock resolver +options INVARIANTS # Enable calls of extra sanity checking +options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +options WITNESS # Enable checks to detect deadlocks and cycles +options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones + +# To make an SMP kernel, the next two lines are needed +options SMP # Symmetric MultiProcessor Kernel +device apic # I/O APIC + +# CPU frequency control +device cpufreq + +# Bus support. +device acpi +device pci +options PCI_HP # PCI-Express native HotPlug +options PCI_IOV # PCI SR-IOV support + +# Floppy drives +device fdc + +# ATA controllers +device ahci # AHCI-compatible SATA controllers +device ata # Legacy ATA/SATA controllers +device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +device ahc # AHA2940 and onboard AIC7xxx devices +options AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +device ahd # AHA39320/29320 and onboard AIC79xx devices +options AHD_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~215k to driver. +device esp # AMD Am53C974 (Tekram DC-390(T)) +device hptiop # Highpoint RocketRaid 3xxx series +device isp # Qlogic family +#device ispfw # Firmware for QLogic HBAs- normally a module +device mpt # LSI-Logic MPT-Fusion +device mps # LSI-Logic MPT-Fusion 2 +device mpr # LSI-Logic MPT-Fusion 3 +#device ncr # NCR/Symbios Logic +device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +device trm # Tekram DC395U/UW/F DC315U adapters + +device adv # Advansys SCSI adapters +device adw # Advansys wide SCSI adapters +device aha # Adaptec 154x SCSI adapters +device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +device bt # Buslogic/Mylex MultiMaster SCSI adapters + +device ncv # NCR 53C500 +device nsp # Workbit Ninja SCSI-3 +device stg # TMC 18C30/18C50 +device isci # Intel C600 SAS controller + +# ATA/SCSI peripherals +device scbus # SCSI bus (required for ATA/SCSI) +device ch # SCSI media changers +device da # Direct Access (disks) +device sa # Sequential Access (tape etc) +device cd # CD +device pass # Passthrough device (direct ATA/SCSI access) +device ses # Enclosure Services (SES and SAF-TE) +#device ctl # CAM Target Layer + +# RAID controllers interfaced to the SCSI subsystem +device amr # AMI MegaRAID +device arcmsr # Areca SATA II RAID +device ciss # Compaq Smart RAID 5* +device dpt # DPT Smartcache III, IV - See NOTES for options +device hptmv # Highpoint RocketRAID 182x +device hptnr # Highpoint DC7280, R750 +device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +device hpt27xx # Highpoint RocketRAID 27xx +device iir # Intel Integrated RAID +device ips # IBM (Adaptec) ServeRAID +device mly # Mylex AcceleRAID/eXtremeRAID +device twa # 3ware 9000 series PATA/SATA RAID +device tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +device aac # Adaptec FSA RAID +device aacp # SCSI passthrough for aac (requires CAM) +device aacraid # Adaptec by PMC RAID +device ida # Compaq Smart RAID +device mfi # LSI MegaRAID SAS +device mlx # Mylex DAC960 family +device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s +device pmspcv # PMC-Sierra SAS/SATA Controller driver +device pst # Promise Supertrak SX6000 +device twe # 3ware ATA RAID + +# NVM Express (NVMe) support +device nvme # base NVMe driver +device nvd # expose NVMe namespace as disks, depends on nvme + +# atkbdc0 controls both the keyboard and the PS/2 mouse +device atkbdc # AT keyboard controller +device atkbd # AT keyboard +device psm # PS/2 mouse + +device kbdmux # keyboard multiplexer + +device vga # VGA video card driver +options VESA # Add support for VESA BIOS Extensions (VBE) + +device splash # Splash screen and screen saver support + +# syscons is the default console driver, resembling an SCO console +device sc +options SC_PIXEL_MODE # add support for the raster text mode + +# vt is the new video console driver +device vt +device vt_vga + +device agp # support several AGP chipsets + +# Power management support (see NOTES for more options) +#device apm +# Add suspend/resume support for the i8254. +device pmtimer + +# PCCARD (PCMCIA) support +# PCMCIA and cardbus bridge support +device cbb # cardbus (yenta) bridge +device pccard # PC Card (16-bit) bus +device cardbus # CardBus (32-bit) bus + +# Serial (COM) ports +device uart # Generic UART driver + +# Parallel port +device ppc +device ppbus # Parallel port bus (required) +device lpt # Printer +device ppi # Parallel port interface device +#device vpo # Requires scbus and da + +device puc # Multi I/O cards and multi-channel UARTs + +# PCI Ethernet NICs. +device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +device de # DEC/Intel DC21x4x (``Tulip'') +device em # Intel PRO/1000 Gigabit Ethernet Family +device igb # Intel PRO/1000 PCIE Server Gigabit Family +device ixgb # Intel PRO/10GbE Ethernet Card +device le # AMD Am7900 LANCE and Am79C9xx PCnet +device ti # Alteon Networks Tigon I/II gigabit Ethernet +device txp # 3Com 3cR990 (``Typhoon'') +device vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! +device miibus # MII bus support +device ae # Attansic/Atheros L2 FastEthernet +device age # Attansic/Atheros L1 Gigabit Ethernet +device alc # Atheros AR8131/AR8132 Ethernet +device ale # Atheros AR8121/AR8113/AR8114 Ethernet +device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +device bfe # Broadcom BCM440x 10/100 Ethernet +device bge # Broadcom BCM570xx Gigabit Ethernet +device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +device dc # DEC/Intel 21143 and various workalikes +device et # Agere ET1310 10/100/Gigabit Ethernet +device fxp # Intel EtherExpress PRO/100B (82557, 82558) +device gem # Sun GEM/Sun ERI/Apple GMAC +device hme # Sun HME (Happy Meal Ethernet) +device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +device lge # Level 1 LXT1001 gigabit Ethernet +device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +device nfe # nVidia nForce MCP on-board Ethernet +device nge # NatSemi DP83820 gigabit Ethernet +device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +device re # RealTek 8139C+/8169/8169S/8110S +device rl # RealTek 8129/8139 +device sf # Adaptec AIC-6915 (``Starfire'') +device sge # Silicon Integrated Systems SiS190/191 +device sis # Silicon Integrated Systems SiS 900/SiS 7016 +device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +device ste # Sundance ST201 (D-Link DFE-550TX) +device stge # Sundance/Tamarack TC9021 gigabit Ethernet +device tl # Texas Instruments ThunderLAN +device tx # SMC EtherPower II (83c170 ``EPIC'') +device vge # VIA VT612x gigabit Ethernet +device vr # VIA Rhine, Rhine II +device vte # DM&P Vortex86 RDC R6040 Fast Ethernet +device wb # Winbond W89C840F +device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# ISA Ethernet NICs. pccard NICs included. +device cs # Crystal Semiconductor CS89x0 NIC +# 'device ed' requires 'device miibus' +device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards +device ex # Intel EtherExpress Pro/10 and Pro/10+ +device ep # Etherlink III based cards +device fe # Fujitsu MB8696x based cards +device ie # EtherExpress 8/16, 3C507, StarLAN 10 etc. +device sn # SMC's 9000 series of Ethernet chips +device xe # Xircom pccard Ethernet + +# Wireless NIC cards +device wlan # 802.11 support +options IEEE80211_DEBUG # enable debug msgs +options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's +options IEEE80211_SUPPORT_MESH # enable 802.11s draft support +device wlan_wep # 802.11 WEP support +device wlan_ccmp # 802.11 CCMP support +device wlan_tkip # 802.11 TKIP support +device wlan_amrr # AMRR transmit rate control algorithm +device an # Aironet 4500/4800 802.11 wireless NICs. +device ath # Atheros NICs +device ath_pci # Atheros pci/cardbus glue +device ath_hal # pci/cardbus chip support +options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors +options AH_AR5416_INTERRUPT_MITIGATION # AR5416 interrupt mitigation +options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later +device ath_rate_sample # SampleRate tx rate control for ath +#device bwi # Broadcom BCM430x/BCM431x wireless NICs. +#device bwn # Broadcom BCM43xx wireless NICs. +device ipw # Intel 2100 wireless NICs. +device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +device iwn # Intel 4965/1000/5000/6000 wireless NICs. +device malo # Marvell Libertas wireless NICs. +device mwl # Marvell 88W8363 802.11n wireless NICs. +device ral # Ralink Technology RT2500 wireless NICs. +device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +#device wl # Older non 802.11 Wavelan wireless NIC. +device wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +device loop # Network loopback +device random # Entropy device +device padlock_rng # VIA Padlock RNG +device rdrand_rng # Intel Bull Mountain RNG +device ether # Ethernet support +device vlan # 802.1Q VLAN support +device tun # Packet tunnel. +device md # Memory "disks" +device gif # IPv6 and IPv4 tunneling +device firmware # firmware assist module + +# The `bpf' device enables the Berkeley Packet Filter. +# Be aware of the administrative consequences of enabling this! +# Note that 'bpf' is required for DHCP. +device bpf # Berkeley packet filter + +# USB support +options USB_DEBUG # enable debug msgs +device uhci # UHCI PCI->USB interface +device ohci # OHCI PCI->USB interface +device ehci # EHCI PCI->USB interface (USB 2.0) +device xhci # XHCI PCI->USB interface (USB 3.0) +device usb # USB Bus (required) +device ukbd # Keyboard +device umass # Disks/Mass storage - Requires scbus and da + +# Sound support +device sound # Generic sound driver (required) +device snd_cmi # CMedia CMI8338/CMI8738 +device snd_csa # Crystal Semiconductor CS461x/428x +device snd_emu10kx # Creative SoundBlaster Live! and Audigy +device snd_es137x # Ensoniq AudioPCI ES137x +device snd_hda # Intel High Definition Audio +device snd_ich # Intel, NVidia and other ICH AC'97 Audio +device snd_via8233 # VIA VT8233x Audio + +# MMC/SD +device mmc # MMC/SD bus +device mmcsd # MMC/SD memory card +device sdhci # Generic PCI SD Host Controller + +# VirtIO support +device virtio # Generic VirtIO bus (required) +device virtio_pci # VirtIO PCI device +device vtnet # VirtIO Ethernet device +device virtio_blk # VirtIO Block device +device virtio_scsi # VirtIO SCSI device +device virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers and enchancement support +device hyperv # HyperV drivers + +# Xen HVM Guest Optimizations +# NOTE: XENHVM depends on xenpci. They must be added or removed together. +options XENHVM # Xen HVM kernel infrastructure +device xenpci # Xen HVM Hypervisor services driver + +# VMware support +device vmx # VMware VMXNET3 Ethernet + +# The crypto framework is required by IPSEC +device crypto # Required by IPSEC -- 2.11.0 From 702d8b0f497bf492e93144f5f382c630d4b52013 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 17:55:42 +0200 Subject: [PATCH 006/255] Adjust shiny new ELECTRO_BLOAT i386 for ElectroBSD Obtained from: ElectroBSD --- sys/i386/conf/ELECTRO_BLOAT | 81 ++++++++++----------------------------------- 1 file changed, 17 insertions(+), 64 deletions(-) diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT index aefc50753f0d..fcfb7c314411 100644 --- a/sys/i386/conf/ELECTRO_BLOAT +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -1,27 +1,11 @@ -# -# GENERIC -- Generic kernel configuration file for FreeBSD/i386 -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ - -cpu I486_CPU -cpu I586_CPU +# ELECTRO_BLOAT -- Modified copy of the GENERIC kernel configuration file +# Used for the release media. + cpu I686_CPU -ident GENERIC +ident ELECTRO_BLOAT + +# Hacker-movie-compatible colors +options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support @@ -30,9 +14,7 @@ options SCHED_ULE # ULE scheduler options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols -options IPSEC # IP (v4/v6) security options TCP_OFFLOAD # TCP offload -options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists @@ -49,12 +31,7 @@ options CD9660 # ISO 9660 Filesystem options PROCFS # Process filesystem (requires PSEUDOFS) options PSEUDOFS # Pseudo-filesystem framework options GEOM_PART_GPT # GUID Partition Tables. -options GEOM_RAID # Soft RAID functionality. options GEOM_LABEL # Provides labelization -options COMPAT_FREEBSD4 # Compatible with FreeBSD4 -options COMPAT_FREEBSD5 # Compatible with FreeBSD5 -options COMPAT_FREEBSD6 # Compatible with FreeBSD6 -options COMPAT_FREEBSD7 # Compatible with FreeBSD7 options COMPAT_FREEBSD9 # Compatible with FreeBSD9 options COMPAT_FREEBSD10 # Compatible with FreeBSD10 options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI @@ -85,10 +62,6 @@ options KDB_TRACE # Print a stack trace for a panic. options DDB # Support DDB. options GDB # Support remote GDB. options DEADLKRES # Enable the deadlock resolver -options INVARIANTS # Enable calls of extra sanity checking -options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS -options WITNESS # Enable checks to detect deadlocks and cycles -options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # To make an SMP kernel, the next two lines are needed @@ -101,15 +74,12 @@ device cpufreq # Bus support. device acpi device pci -options PCI_HP # PCI-Express native HotPlug options PCI_IOV # PCI SR-IOV support -# Floppy drives -device fdc - # ATA controllers device ahci # AHCI-compatible SATA controllers device ata # Legacy ATA/SATA controllers +options ATA_STATIC_ID # Static device numbering device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA @@ -123,12 +93,9 @@ options AHD_REG_PRETTY_PRINT # Print register bitfields in debug device esp # AMD Am53C974 (Tekram DC-390(T)) device hptiop # Highpoint RocketRaid 3xxx series device isp # Qlogic family -#device ispfw # Firmware for QLogic HBAs- normally a module device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -#device ncr # NCR/Symbios Logic -device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters device adv # Advansys SCSI adapters @@ -157,10 +124,6 @@ device amr # AMI MegaRAID device arcmsr # Areca SATA II RAID device ciss # Compaq Smart RAID 5* device dpt # DPT Smartcache III, IV - See NOTES for options -device hptmv # Highpoint RocketRAID 182x -device hptnr # Highpoint DC7280, R750 -device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx -device hpt27xx # Highpoint RocketRAID 27xx device iir # Intel Integrated RAID device ips # IBM (Adaptec) ServeRAID device mly # Mylex AcceleRAID/eXtremeRAID @@ -173,16 +136,10 @@ device aacp # SCSI passthrough for aac (requires CAM) device aacraid # Adaptec by PMC RAID device ida # Compaq Smart RAID device mfi # LSI MegaRAID SAS -device mlx # Mylex DAC960 family device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s -device pmspcv # PMC-Sierra SAS/SATA Controller driver device pst # Promise Supertrak SX6000 device twe # 3ware ATA RAID -# NVM Express (NVMe) support -device nvme # base NVMe driver -device nvd # expose NVMe namespace as disks, depends on nvme - # atkbdc0 controls both the keyboard and the PS/2 mouse device atkbdc # AT keyboard controller device atkbd # AT keyboard @@ -205,8 +162,6 @@ device vt_vga device agp # support several AGP chipsets -# Power management support (see NOTES for more options) -#device apm # Add suspend/resume support for the i8254. device pmtimer @@ -229,7 +184,6 @@ device ppi # Parallel port interface device device puc # Multi I/O cards and multi-channel UARTs # PCI Ethernet NICs. -device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family device igb # Intel PRO/1000 PCIE Server Gigabit Family @@ -246,7 +200,6 @@ device ae # Attansic/Atheros L2 FastEthernet device age # Attansic/Atheros L1 Gigabit Ethernet device alc # Atheros AR8131/AR8132 Ethernet device ale # Atheros AR8121/AR8113/AR8114 Ethernet -device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet device bfe # Broadcom BCM440x 10/100 Ethernet device bge # Broadcom BCM570xx Gigabit Ethernet device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn @@ -307,15 +260,7 @@ options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later device ath_rate_sample # SampleRate tx rate control for ath #device bwi # Broadcom BCM430x/BCM431x wireless NICs. #device bwn # Broadcom BCM43xx wireless NICs. -device ipw # Intel 2100 wireless NICs. -device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. -device iwn # Intel 4965/1000/5000/6000 wireless NICs. device malo # Marvell Libertas wireless NICs. -device mwl # Marvell 88W8363 802.11n wireless NICs. -device ral # Ralink Technology RT2500 wireless NICs. -device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. -#device wl # Older non 802.11 Wavelan wireless NIC. -device wpi # Intel 3945ABG wireless NICs. # Pseudo devices. device loop # Network loopback @@ -347,7 +292,6 @@ device umass # Disks/Mass storage - Requires scbus and da # Sound support device sound # Generic sound driver (required) device snd_cmi # CMedia CMI8338/CMI8738 -device snd_csa # Crystal Semiconductor CS461x/428x device snd_emu10kx # Creative SoundBlaster Live! and Audigy device snd_es137x # Ensoniq AudioPCI ES137x device snd_hda # Intel High Definition Audio @@ -380,3 +324,12 @@ device vmx # VMware VMXNET3 Ethernet # The crypto framework is required by IPSEC device crypto # Required by IPSEC + +# This used to be required for ZFS when compiled with clang. +# For details see UPDATING entry 20121223. After r286288 it's +# probably no longer necessary, but for now we keep it anyway. +options KSTACK_PAGES=4 + +# Increase the size of the kernel virtual address space +# so ZFS can cache more stuff. +options KVA_PAGES=512 -- 2.11.0 From ef2edb33ed0431a6a7096c6c1d8c7430f716d21f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:38:09 +0200 Subject: [PATCH 007/255] Change amd64 default KERNCONF to ELECTRO_BLOAT Obtained from: ElectroBSD --- Makefile.inc1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index 6d96cead7f19..d930bbd4eb2f 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1136,6 +1136,8 @@ KERNCONF=${KERNFAST} .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 +.elif ${TARGET_ARCH} == "amd64" +KERNCONF?= ELECTRO_BLOAT .else KERNCONF?= GENERIC .endif -- 2.11.0 From 121baa9398d496b5007b3b2dbe7e7e3de96948a8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 22 May 2016 12:10:03 +0200 Subject: [PATCH 008/255] Use ELECTRO_BLOAT as default on i386 Obtained from: ElectroBSD --- Makefile.inc1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index d930bbd4eb2f..dc1d0a440be9 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1136,7 +1136,7 @@ KERNCONF=${KERNFAST} .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 -.elif ${TARGET_ARCH} == "amd64" +.elif ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" KERNCONF?= ELECTRO_BLOAT .else KERNCONF?= GENERIC -- 2.11.0 From 9f30776c12bafedace9045434e0f9c84b1a9328d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Aug 2011 19:19:51 +0200 Subject: [PATCH 009/255] Register the product id for Feiya Memory Bar Obtained from: ElectroBSD --- sys/dev/usb/usbdevs | 1 + 1 file changed, 1 insertion(+) diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index fc898ffca6e9..1c6cf7620d27 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -1867,6 +1867,7 @@ product FALCOM SAMBA 0x0005 FTDI compatible adapter /* FEIYA products */ product FEIYA DUMMY 0x0000 Dummy product +product FEIYA MEMORY_BAR 0x1000 Memory Bar product FEIYA 5IN1 0x1132 5-in-1 Card Reader product FEIYA ELANGO 0x6200 MicroSDHC Card Reader product FEIYA AC110 0x6300 AC-110 Card Reader -- 2.11.0 From 69e8213cbe5c581c0b594e8b11ba85a7a7a6af29 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Aug 2011 19:22:46 +0200 Subject: [PATCH 010/255] Add the full name of 'Feya Technology Corp.' Obtained from: ElectroBSD --- sys/dev/usb/usbdevs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index 1c6cf7620d27..93d7a7a88cd0 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -446,7 +446,7 @@ vendor DIGIANSWER 0x08fd Digianswer vendor AUTHENTEC 0x08ff AuthenTec vendor AUDIOTECHNICA 0x0909 Audio-Technica vendor TRUMPION 0x090a Trumpion Microelectronics -vendor FEIYA 0x090c Feiya +vendor FEIYA 0x090c Feiya Technology Corp. vendor ALATION 0x0910 Alation Systems vendor GLOBESPAN 0x0915 Globespan vendor CONCORDCAMERA 0x0919 Concord Camera -- 2.11.0 From 223a87e56536376542d63cd2082d4c00b70e2bc5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 Feb 2013 14:42:00 +0100 Subject: [PATCH 011/255] Let g_eli_*read_done() deal with lost devices without causing panics Seems to fix kern/162036 for me. Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 3 ++- sys/geom/eli/g_eli_privacy.c | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 6d734aece183..bd198c043c2c 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -214,7 +214,8 @@ g_eli_read_done(struct bio *bp) pbp->bio_driver2 = NULL; } g_io_deliver(pbp, pbp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); return; } mtx_lock(&sc->sc_queue_mtx); diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c index d636e1fb0ba0..6ed584684092 100644 --- a/sys/geom/eli/g_eli_privacy.c +++ b/sys/geom/eli/g_eli_privacy.c @@ -87,7 +87,8 @@ g_eli_crypto_read_done(struct cryptop *crp) bp->bio_error = crp->crp_etype; } sc = bp->bio_to->geom->softc; - g_eli_key_drop(sc, crp->crp_desc->crd_key); + if (sc != NULL) + g_eli_key_drop(sc, crp->crp_desc->crd_key); /* * Do we have all sectors already? */ @@ -104,7 +105,8 @@ g_eli_crypto_read_done(struct cryptop *crp) * Read is finished, send it up. */ g_io_deliver(bp, bp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); return (0); } -- 2.11.0 From 8cffc55b4479fce90d612e4896144a423a332195 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 Feb 2013 11:09:20 +0100 Subject: [PATCH 012/255] Let g_eli_write_done() verify that sc isn't NULL as well It looks similar enough to g_eli_read_done() to be affected by kern/162036 as well. Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index bd198c043c2c..920ed0cc118e 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -260,7 +260,8 @@ g_eli_write_done(struct bio *bp) */ sc = pbp->bio_to->geom->softc; g_io_deliver(pbp, pbp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); } /* -- 2.11.0 From 2d069a4c9beecd3f3bec5ccfd19f88de0e06345c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 20 Dec 2013 18:45:00 +0100 Subject: [PATCH 013/255] Let g_eli_keyfiles_load() log the size of the keyfile loaded Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 920ed0cc118e..9add13044a6d 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -959,8 +959,8 @@ g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider) name); return (0); } - G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file, - provider, name); + G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s) (size: %d).", file, + provider, name, (unsigned)size); g_eli_crypto_hmac_update(ctx, data, size); } } -- 2.11.0 From 2036699a3077a12170c1ead5fd9bc883d483f8bc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 4 May 2014 23:19:04 +0200 Subject: [PATCH 014/255] newvers.sh: Skip dirty tree detection. Too fucking slow Obtained from: ElectroBSD --- sys/conf/newvers.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 0d5f795e1f26..a1539532eb01 100644 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -193,10 +193,10 @@ if [ -n "$git_cmd" ] ; then if [ -n "$git_b" ] ; then git="${git}(${git_b})" fi - if $git_cmd --work-tree=${SYSDIR}/.. diff-index \ - --name-only HEAD | read dummy; then - git="${git}-dirty" - fi +# if $git_cmd --work-tree=${SYSDIR}/.. diff-index \ +# --name-only HEAD | read dummy; then +# git="${git}-dirty" +# fi fi if [ -n "$p4_cmd" ] ; then -- 2.11.0 From e63539206671de0fefe985a69be48711800be304 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 13:27:28 +0100 Subject: [PATCH 015/255] ggated: Ignore SIGPIPE to prevent DoS ... by a single prematurely closed client connection. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index e234cb57b290..36a8d5203b21 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1025,6 +1025,7 @@ main(int argc, char *argv[]) pidfile_write(pfh); signal(SIGCHLD, SIG_IGN); + signal(SIGPIPE, SIG_IGN); sfd = socket(AF_INET, SOCK_STREAM, 0); if (sfd == -1) -- 2.11.0 From f57f2f9b3a9523d5ab86d2b3e4613e1bd8767a80 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 13:47:31 +0100 Subject: [PATCH 016/255] ggated: Remove connection if the initial packet couldn't be sent Should help to mitigate DoS after flooding ggated with incomplete requests: error: accept(): Software caused connection abort. error: Exiting. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 36a8d5203b21..7574e20d3480 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -928,6 +928,7 @@ handshake(struct sockaddr *from, int sfd) if (data == -1) { sendfail(sfd, errno, "Error while sending initial packet: %s.", strerror(errno)); + connection_remove(conn); return (0); } -- 2.11.0 From 675ad84cd54aca8d6112e9e23b5261fa46558833 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 14:09:24 +0100 Subject: [PATCH 017/255] ggated: Continue if accept() is interrupted or the remote connection is lost Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 7574e20d3480..fc7c86ae75ac 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1050,9 +1050,11 @@ main(int argc, char *argv[]) for (;;) { fromlen = sizeof(from); tmpsfd = accept(sfd, &from, &fromlen); - if (tmpsfd == -1) + if (tmpsfd == -1) { + if (errno == EINTR || errno == ECONNABORTED) + continue; g_gate_xlog("accept(): %s.", strerror(errno)); - + } if (got_sighup) { got_sighup = 0; exports_get(); -- 2.11.0 From 4cef444484d0bee4c534e90ac18c1ccfd3ff74d8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 15:18:27 +0100 Subject: [PATCH 018/255] ggated: Initialize conn->c_diskfd in connection_new() Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index fc7c86ae75ac..9d523459623d 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -453,7 +453,7 @@ connection_new(struct g_gate_cinit *cinit, struct sockaddr *s, int sfd) conn->c_token = cinit->gc_token; ip = htonl(((struct sockaddr_in *)(void *)s)->sin_addr.s_addr); conn->c_srcip = ip; - conn->c_sendfd = conn->c_recvfd = -1; + conn->c_sendfd = conn->c_recvfd = conn->c_diskfd = -1; if ((cinit->gc_flags & GGATE_FLAG_SEND) != 0) conn->c_sendfd = sfd; else -- 2.11.0 From 43fc3d67c00782e0824025c9920c99f2adf27a27 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 14:16:47 +0100 Subject: [PATCH 019/255] ggated: Prevent c_diskfd leaks through connection_remove() Should help against DoS: [...] debug: Connection created [127.0.0.1, /tank/scratch/testfile]. debug: New connection created (token=2197914058). debug: exports[/tank/scratch/testfile2]: Path mismatch. debug: Sending initial packet. error: accept(): Too many open files. error: Exiting. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 9d523459623d..6d5521804e60 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -512,6 +512,8 @@ connection_remove(struct ggd_connection *conn) close(conn->c_sendfd); if (conn->c_recvfd != -1) close(conn->c_recvfd); + if (conn->c_diskfd != -1) + close(conn->c_diskfd); free(conn->c_path); free(conn); } -- 2.11.0 From d1147f7bb107d070432b832506ed5251ee0dcf80 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 15:52:39 +0100 Subject: [PATCH 020/255] ggated: Check for connection_add() failures properly Prevents a socket leak Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6d5521804e60..6236c50a4359 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -890,7 +890,7 @@ handshake(struct sockaddr *from, int sfd) */ g_gate_log(LOG_DEBUG, "Found existing connection (token=%lu).", (unsigned long)conn->c_token); - if (connection_add(conn, &cinit, from, sfd) == -1) { + if (connection_add(conn, &cinit, from, sfd) == EEXIST) { connection_remove(conn); return (0); } -- 2.11.0 From b76ffaf9cf9a8de2143656b22cee3919ce2c7683 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 8 Dec 2014 17:59:38 +0100 Subject: [PATCH 021/255] ggated: Do not leak stack data in sendfail() Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6236c50a4359..4bb1f7c6004a 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -591,6 +591,7 @@ sendfail(int sfd, int error, const char *fmt, ...) va_list ap; ssize_t data; + bzero(&sinit, sizeof(sinit)); sinit.gs_error = error; g_gate_swap2n_sinit(&sinit); data = g_gate_send(sfd, &sinit, sizeof(sinit), 0); -- 2.11.0 From 6dd56eda5630668e95d9657dabbdb9d0f66cb6a3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 15:24:58 +0200 Subject: [PATCH 022/255] ggated recv_thread(): Do not queue incomplete WRITE requests Verifying that g_gate_recv()'s return code isn't -1 is insufficient as it's a thin wrapper arround recv(2) which, quoting its man page, "may still return less data than requested if a signal is caught, an error or disconnect occurs, or the next data to be received is of a different type than that returned". Previously incomplete WRITE requests would be scheduled with partially uninitialized memory, potentially resulting in file system corruption or, worse, bogus data being later on returned as valid. Security impact: A MITM may cause data corruption by disrupting the connection from ggatec's send_thread() to ggated's recv_thread() at the right point in time. This does not require access to the plain text traffic but if encryption is involved the attacker would have to guess that it's ggate traffic and disrupt connections blindly, hoping that some of the disruptions trigger the bug. The issue was discovered after ZFS on the ggatec side reported checksum errors which weren't reproducible on the ggated side where ZFS had received and checksummed bogus data. The ggate traffic was tunneled through SSH and Tor with sshd running as Tor location hidden service. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 4bb1f7c6004a..9883355d81f1 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -669,6 +669,9 @@ recv_thread(void *arg) if (data == -1) { g_gate_xlog("Error while receiving data: %s.", strerror(errno)); + } else if ((uint32_t)data != req->r_length) { + g_gate_xlog("Received %d bytes of data while " + "expecting %u.", data, req->r_length); } } -- 2.11.0 From b7ed4c9ae46330435d3812febcea05b075b9231e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 12:09:40 +0200 Subject: [PATCH 023/255] ggated recv_thread(): Do not queue requests with invalid values ... that would cause abort()s when read by the disk_thread() later on. From ggatec's point of view it doesn't make a difference as the connection will get closed either way, but at least the admin on the server side doesn't have to deal with core dumps. Security impact: An authenticated attacker may intentionally cause the ggated process that handles the attacker's connection to core dump and thus use more disk space than intentionally provisioned by the server admin. Without the following patch ggated core dumps may require more than 100 GB of disk space. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 9883355d81f1..6b53decdeb65 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -654,6 +654,23 @@ recv_thread(void *arg) (intmax_t)req->r_offset, (unsigned)req->r_length); /* + * Reject requests that violate assertions in disk_thread(). + */ + if (req->r_cmd != GGATE_CMD_READ && + req->r_cmd != GGATE_CMD_WRITE) { + g_gate_xlog("Request contains invalid command."); + } + if (req->r_offset + req->r_length > + (uintmax_t)conn->c_mediasize) { + g_gate_xlog("Request out of bounds."); + } + if (req->r_offset % conn->c_sectorsize != 0 || + req->r_length % conn->c_sectorsize != 0) { + g_gate_xlog("Request length or offset does " + "not fit sector size."); + } + + /* * Allocate memory for data. */ req->r_data = malloc_waitok(req->r_length); -- 2.11.0 From 5029d99fc8c849540ed410db33507b0d77c5d2f5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 19:52:54 +0200 Subject: [PATCH 024/255] ggated recv_thread(): Reject request with more than MAXPHYS bytes of data .. to limit the amount of memory we (try to) allocate on behalf of the client without knowing whether or not the client actually intents to use it. MAXPHYS is the hardcoded limit in ggatec so anything above it is suspicious and could be a DoS attempt. This commit forces users who like to tune MAXPHYS to make sure the value used by ggated is not below the one used by ggatec. While not ideal, this seems preferable to the DoS risk. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6b53decdeb65..22810b0a00fd 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -671,6 +671,16 @@ recv_thread(void *arg) } /* + * Limit the amount of memory we allocate on behalf of + * the client. MAXPHYS is the hard limit in ggatec, + * values above it are thus pretty suspicious. + */ + if (req->r_length > MAXPHYS) { + g_gate_xlog("Request length above MAXPHYS: %u > %u", + (unsigned)req->r_length, MAXPHYS); + } + + /* * Allocate memory for data. */ req->r_data = malloc_waitok(req->r_length); -- 2.11.0 From f93378cc8996e03956d748f187739b2ec03c81de Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Apr 2015 14:04:31 +0200 Subject: [PATCH 025/255] ggatec: Add support for SOCKS5 with domain names Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 104 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 5 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ea4f701abafa..80a653dff6ba 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -64,6 +64,8 @@ static unsigned flags = 0; static int force = 0; static unsigned queue_size = G_GATE_QUEUE_SIZE; static unsigned port = G_GATE_PORT; +static char *socks_dest = NULL; +static unsigned dest_port = 3080; static off_t mediasize; static unsigned sectorsize = 0; static unsigned timeout = G_GATE_TIMEOUT; @@ -78,9 +80,11 @@ usage(void) fprintf(stderr, "usage: %s create [-nv] [-o ] [-p port] " "[-q queue_size] [-R rcvbuf] [-S sndbuf] [-s sectorsize] " - "[-t timeout] [-u unit] \n", getprogname()); + "[-t timeout] [-T :] [-u unit] \n", + getprogname()); fprintf(stderr, " %s rescue [-nv] [-o ] [-p port] " - "[-R rcvbuf] [-S sndbuf] <-u unit> \n", getprogname()); + "[-R rcvbuf] [-S sndbuf] [-T :] <-u unit> " + " \n", getprogname()); fprintf(stderr, " %s destroy [-f] <-u unit>\n", getprogname()); fprintf(stderr, " %s list [-v] [-u unit]\n", getprogname()); exit(EXIT_FAILURE); @@ -238,6 +242,69 @@ recv_thread(void *arg __unused) pthread_exit(NULL); } +static void +negotiate_socks_connection(int sfd) +{ + struct negotiation_request { + char version; + char nmethods; + char method; + } neg_request; + struct socks_request { + char version; + char cmd; + char reserved; + char address_type; + char host_length; + char dest[255 + 2]; + } socks_request; + char response[10]; + size_t request_length; + size_t host_length; + + host_length = strlen(socks_dest); + + neg_request.version = '\x05'; + neg_request.nmethods = '\x01'; /* We support one method: */ + neg_request.method = '\x00'; /* no authentication */ + + g_gate_log(LOG_DEBUG, "Starting SOCKS negotiation."); + if (g_gate_send(sfd, &neg_request, sizeof(neg_request), MSG_NOSIGNAL) == -1) + g_gate_xlog("Failed to send SOCKS negotiation request."); + + if (g_gate_recv(sfd, &response, sizeof(response), MSG_WAITALL) != 2) + g_gate_xlog("Failed to read SOCKS negotiation response."); + + if (response[0] != '\x05' || response[1] != '\x00') + g_gate_xlog("SOCKS negotiation failed."); + + g_gate_log(LOG_DEBUG, "Negotiated SOCKS5. " + "Requesting connection to %s:%d.", socks_dest, dest_port); + + socks_request.version = '\x05'; + socks_request.cmd = '\x01'; /* Connect */ + socks_request.reserved = '\x00'; + socks_request.address_type = '\x03'; /* Address is domain name */; + socks_request.host_length = (char)host_length; + strncpy(socks_request.dest, socks_dest, host_length); + socks_request.dest[host_length] = (char)((dest_port >> 8) & 0xff); + socks_request.dest[host_length + 1] = (char)(dest_port & 0xff); + request_length = sizeof(socks_request) - sizeof(socks_request.dest) + + host_length + 2; + + if (g_gate_send(sfd, &socks_request, request_length, MSG_NOSIGNAL) == -1) + g_gate_xlog("Failed to send SOCKS5 request."); + + if (g_gate_recv(sfd, &response, sizeof(response), MSG_WAITALL) != sizeof(response)) + g_gate_xlog("Failed to read SOCKS5 response."); + + if (response[0] != '\x05' || response[1] != '\x00') + g_gate_xlog("Failed to SOCKS5 connect to %s:%d", + socks_dest, dest_port); + + g_gate_log(LOG_INFO, "Connected to: %s:%d.", socks_dest, dest_port); +} + static int handshake(int dir) { @@ -276,6 +343,9 @@ handshake(int dir) g_gate_log(LOG_INFO, "Connected to the server: %s:%d.", host, port); + if (socks_dest != NULL) + negotiate_socks_connection(sfd); + /* * Create and send version packet. */ @@ -455,8 +525,13 @@ g_gatec_create(void) ggioc.gctl_maxcount = queue_size; ggioc.gctl_timeout = timeout; ggioc.gctl_unit = unit; - snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), "%s:%u %s", host, - port, path); + if (socks_dest != NULL) + snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), + "socks5://%s:%u -> %s:%u %s", host, + port, socks_dest, dest_port, path); + else + snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), "%s:%u %s", + host, port, path); g_gate_ioctl(G_GATE_CMD_CREATE, &ggioc); if (unit == -1) { printf("%s%u\n", G_GATE_PROVIDER_NAME, ggioc.gctl_unit); @@ -505,8 +580,9 @@ main(int argc, char *argv[]) argv += 1; for (;;) { int ch; + char *p; - ch = getopt(argc, argv, "fno:p:q:R:S:s:t:u:v"); + ch = getopt(argc, argv, "fno:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { @@ -574,6 +650,24 @@ main(int argc, char *argv[]) if (sectorsize == 0 && errno != 0) errx(EXIT_FAILURE, "Invalid sectorsize."); break; + case 'T': + if (action != CREATE && action != RESCUE) + usage(); + socks_dest = optarg; + p = strchr(socks_dest, ':'); + if (p != NULL) { + errno = 0; + *p = '\0'; + p++; + dest_port = strtoul(p, NULL, 10); + if (dest_port == 0 && errno != 0) + errx(EXIT_FAILURE, + "Invalid socks5t port: %s.", p); + } + if (strlen(socks_dest) > (size_t)255) + errx(EXIT_FAILURE, + "Socks destination address too long."); + break; case 't': if (action != CREATE) usage(); -- 2.11.0 From b7e6fa1811145588469f9910711c0a4ec8bc0076 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 16:44:32 +0200 Subject: [PATCH 026/255] ggatec: Deduplicate information in debug output Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 80a653dff6ba..ed5b94915ac6 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -176,8 +176,9 @@ send_thread(void *arg __unused) pthread_kill(recvtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Sent %zd bytes (offset=%llu, " - "size=%u).", data, hdr.gh_offset, hdr.gh_length); + g_gate_log(LOG_DEBUG, "Sent data packet " + "(offset=%llu, size=%u).", + hdr.gh_offset, hdr.gh_length); } } g_gate_log(LOG_DEBUG, "%s: Died.", __func__); @@ -224,16 +225,15 @@ recv_thread(void *arg __unused) ggio.gctl_length, MSG_WAITALL); if (reconnect) break; - g_gate_log(LOG_DEBUG, "Received data packet."); if (data != ggio.gctl_length) { g_gate_log(LOG_ERR, "Lost connection 4."); reconnect = 1; pthread_kill(sendtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Received %d bytes (offset=%ju, " - "size=%zu).", data, (uintmax_t)hdr.gh_offset, - (size_t)hdr.gh_length); + g_gate_log(LOG_DEBUG, "Received data packet " + "(offset=%ju, size=%zu).", + (uintmax_t)hdr.gh_offset, (size_t)hdr.gh_length); } g_gate_ioctl(G_GATE_CMD_DONE, &ggio); -- 2.11.0 From 7b1b711ec4040b9216b8db4150ccdb87b386e8a4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Apr 2015 15:26:42 +0200 Subject: [PATCH 027/255] ggatec.8: Document SOCKS5 support Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.8 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.8 b/sbin/ggate/ggatec/ggatec.8 index 705d42f657c2..c50416f4e418 100644 --- a/sbin/ggate/ggatec/ggatec.8 +++ b/sbin/ggate/ggatec/ggatec.8 @@ -41,6 +41,7 @@ .Op Fl R Ar rcvbuf .Op Fl S Ar sndbuf .Op Fl s Ar sectorsize +.Op Fl T Ar remote_target:port .Op Fl t Ar timeout .Op Fl u Ar unit .Ar host @@ -53,6 +54,7 @@ .Op Fl p Ar port .Op Fl R Ar rcvbuf .Op Fl S Ar sndbuf +.Op Fl T Ar remote_target:port .Fl u Ar unit .Ar host .Ar path @@ -137,6 +139,9 @@ Sector size for .Nm ggate provider. If not specified, it is taken from the device, or set to 512 bytes for files. +.It Fl T Ar remote_host:port +Use SOCK5 to open connection to remote_host:port before switching +to the ggated protocol. .It Fl t Ar timeout Number of seconds to wait before an I/O request will be canceled. Default is 0, which means no timeout. @@ -167,6 +172,14 @@ server# ggated client# ggatec create -o ro server /dev/acd0 ggate0 client# mount_cd9660 /dev/ggate0 /cdrom + +.Ed +Connect to 127.0.1.1:9050, SOCKS5-negotiate a connection to +the Tor location hidden service czdqtfrgvizltdal.onion:1312 +and access a ZVOL: +.Bd -literal -offset indent +# ggatec create -T czdqtfrgvizltdal.onion:1312 -p 9050 \\ + 127.0.1.1 /dev/zvol/dpool/ggated/czdqtfrgvizltdal.eli .Ed .Sh SEE ALSO .Xr geom 4 , -- 2.11.0 From 94d113f9b7d782e00098d4e9d9790cc597204ee2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:10:17 +0200 Subject: [PATCH 028/255] ggatec: Reject unexpected GGATE commands in recv_thread() Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ed5b94915ac6..67a0f405047e 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -220,6 +220,11 @@ recv_thread(void *arg __unused) ggio.gctl_length = hdr.gh_length; ggio.gctl_error = hdr.gh_error; + if (ggio.gctl_cmd != GGATE_CMD_READ && + ggio.gctl_cmd != GGATE_CMD_WRITE) { + g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); + } + if (ggio.gctl_error == 0 && ggio.gctl_cmd == GGATE_CMD_READ) { data = g_gate_recv(recvfd, ggio.gctl_data, ggio.gctl_length, MSG_WAITALL); -- 2.11.0 From 44bddfdb4cfd878b90931d1861ba6b38661fdb58 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:15:18 +0200 Subject: [PATCH 029/255] ggatec: Log if the remote side signals errors Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 67a0f405047e..dd75c6468e9f 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -225,6 +225,12 @@ recv_thread(void *arg __unused) g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } + if (ggio.gctl_error != 0) { + g_gate_log(LOG_ERR, + "Remote side signaled error %d: %s.", + ggio.gctl_error, strerror(ggio.gctl_error)); + } + if (ggio.gctl_error == 0 && ggio.gctl_cmd == GGATE_CMD_READ) { data = g_gate_recv(recvfd, ggio.gctl_data, ggio.gctl_length, MSG_WAITALL); -- 2.11.0 From 61231d1a32168171f8449f7984efe64e482efc5b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:39:25 +0200 Subject: [PATCH 030/255] ggatec: Reject unsupported BIO commands Due to missing sanity checks in ggatec, requests like BIO_FLUSH and BIO_DELETE were previously accepted and sent as zero-size reads or writes. Also due to missing sanity checks, ggated did not mind. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index dd75c6468e9f..650e4f585d54 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,13 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + default: + g_gate_log(LOG_ERR, + "Rejecting unsupported BIO command: %d", + ggio.gctl_cmd); + ggio.gctl_error = EOPNOTSUPP; + g_gate_ioctl(G_GATE_CMD_DONE, &ggio); + continue; } hdr.gh_seq = ggio.gctl_seq; hdr.gh_offset = ggio.gctl_offset; -- 2.11.0 From ea1a9c01f7fdeb4c11566ff6ed0e09d680bcf16e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:53:30 +0200 Subject: [PATCH 031/255] ggate[cd]: Add BIO_FLUSH support Let ggated transform BIO_FLUSH requests into fsync() calls. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 8 +++++++- sbin/ggate/ggated/ggated.c | 23 ++++++++++++++++++----- sbin/ggate/shared/ggate.h | 1 + 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 650e4f585d54..f91c172effb2 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,11 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + case BIO_FLUSH: + g_gate_log(LOG_DEBUG, "FLUSH request"); + hdr.gh_cmd = GGATE_CMD_FLUSH; + assert(ggio.gctl_length == 0); + break; default: g_gate_log(LOG_ERR, "Rejecting unsupported BIO command: %d", @@ -228,7 +233,8 @@ recv_thread(void *arg __unused) ggio.gctl_error = hdr.gh_error; if (ggio.gctl_cmd != GGATE_CMD_READ && - ggio.gctl_cmd != GGATE_CMD_WRITE) { + ggio.gctl_cmd != GGATE_CMD_WRITE && + ggio.gctl_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 22810b0a00fd..2adbb68f7a83 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -657,8 +657,10 @@ recv_thread(void *arg) * Reject requests that violate assertions in disk_thread(). */ if (req->r_cmd != GGATE_CMD_READ && - req->r_cmd != GGATE_CMD_WRITE) { - g_gate_xlog("Request contains invalid command."); + req->r_cmd != GGATE_CMD_WRITE && + req->r_cmd != GGATE_CMD_FLUSH) { + g_gate_xlog("Request contains invalid command: %d", + req->r_cmd); } if (req->r_offset + req->r_length > (uintmax_t)conn->c_mediasize) { @@ -681,9 +683,10 @@ recv_thread(void *arg) } /* - * Allocate memory for data. + * Allocate memory for data, except when flushing. */ - req->r_data = malloc_waitok(req->r_length); + req->r_data = req->r_cmd != GGATE_CMD_FLUSH ? + malloc_waitok(req->r_length) : NULL; /* * Receive data to write for WRITE request. @@ -743,7 +746,9 @@ disk_thread(void *arg) /* * Check the request. */ - assert(req->r_cmd == GGATE_CMD_READ || req->r_cmd == GGATE_CMD_WRITE); + assert(req->r_cmd == GGATE_CMD_READ || + req->r_cmd == GGATE_CMD_WRITE || + req->r_cmd == GGATE_CMD_FLUSH); assert(req->r_offset + req->r_length <= (uintmax_t)conn->c_mediasize); assert((req->r_offset % conn->c_sectorsize) == 0); assert((req->r_length % conn->c_sectorsize) == 0); @@ -767,6 +772,14 @@ disk_thread(void *arg) free(req->r_data); req->r_data = NULL; break; + case GGATE_CMD_FLUSH: + g_gate_log(LOG_DEBUG, "Flushing"); + if (fsync(fd)) { + req->r_error = errno; + g_gate_log(LOG_ERR, "Flushing failed: %s", + strerror(errno)); + } + break; } if (data != (ssize_t)req->r_length) { /* Report short reads/writes as I/O errors. */ diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 898efea6507e..52985660def6 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -55,6 +55,7 @@ #define GGATE_CMD_READ 0 #define GGATE_CMD_WRITE 1 +#define GGATE_CMD_FLUSH 2 extern int g_gate_devfd; extern int g_gate_verbose; -- 2.11.0 From aaf1b4b7d3aca3541bcc4e25ea034ee34b1a72cb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 29 Apr 2015 12:44:56 +0200 Subject: [PATCH 032/255] ggatec: Log the command type for hdr packets (when debugging) ... and provide more details about failed requests. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 11 ++++++++--- sbin/ggate/shared/ggate.c | 16 ++++++++++++++++ sbin/ggate/shared/ggate.h | 1 + 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index f91c172effb2..4d0b12f940ee 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -166,7 +166,9 @@ send_thread(void *arg __unused) g_gate_swap2n_hdr(&hdr); data = g_gate_send(sendfd, &hdr, sizeof(hdr), MSG_NOSIGNAL); - g_gate_log(LOG_DEBUG, "Sent hdr packet."); + g_gate_log(LOG_DEBUG, "Sent hdr packet (%s).", + g_gate_cmd2str(hdr.gh_cmd)); + g_gate_swap2h_hdr(&hdr); if (reconnect) break; @@ -224,7 +226,8 @@ recv_thread(void *arg __unused) pthread_kill(sendtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Received hdr packet."); + g_gate_log(LOG_DEBUG, "Received hdr packet (%s).", + g_gate_cmd2str(hdr.gh_cmd)); ggio.gctl_seq = hdr.gh_seq; ggio.gctl_cmd = hdr.gh_cmd; @@ -240,7 +243,9 @@ recv_thread(void *arg __unused) if (ggio.gctl_error != 0) { g_gate_log(LOG_ERR, - "Remote side signaled error %d: %s.", + "%s for %d bytes at offset %d failed. " + "Error %d: %s.", g_gate_cmd2str(ggio.gctl_cmd), + ggio.gctl_length, ggio.gctl_offset, ggio.gctl_error, strerror(ggio.gctl_error)); } diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index cf9b9ca1574f..05fc25a9afc0 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -407,3 +407,19 @@ g_gate_str2ip(const char *str) return (INADDR_NONE); return (((struct in_addr *)(void *)hp->h_addr)->s_addr); } + +const char * +g_gate_cmd2str(int cmd) +{ + + switch (cmd) { + case GGATE_CMD_READ: + return ("GGATE_CMD_READ"); + case GGATE_CMD_WRITE: + return ("GGATE_CMD_WRITE"); + case GGATE_CMD_FLUSH: + return ("GGATE_CMD_FLUSH"); + } + + return ("unknown (invalid?) GGATE command"); +} diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 52985660def6..36565192a1a8 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -112,6 +112,7 @@ void g_gate_socket_settings(int sfd); void g_gate_list(int unit, int verbose); #endif in_addr_t g_gate_str2ip(const char *str); +const char *g_gate_cmd2str(int cmd); /* * g_gate_swap2h_* - functions swap bytes to host byte order (from big endian). -- 2.11.0 From fa286425de8589c1aca5f248f301406366b7ed83 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 6 May 2015 15:55:08 +0200 Subject: [PATCH 033/255] ggated disk_thread(): Include the command in the debug output Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 2adbb68f7a83..fa11d8b90d13 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -753,7 +753,8 @@ disk_thread(void *arg) assert((req->r_offset % conn->c_sectorsize) == 0); assert((req->r_length % conn->c_sectorsize) == 0); - g_gate_log(LOG_DEBUG, "%s: offset=%jd length=%u", __func__, + g_gate_log(LOG_DEBUG, "%s: cmd=%s offset=%jd length=%u", + __func__, g_gate_cmd2str(req->r_cmd), (intmax_t)req->r_offset, (unsigned)req->r_length); /* -- 2.11.0 From ee8d929c44b7b60c81f8a3c395def6a2f6a5baf0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 5 May 2015 17:39:16 +0200 Subject: [PATCH 034/255] ggate[cd]: Add BIO_DELETE support On the ggated side the requests are translated into writes of zero which ZFS will convert into BIO_DELETE requests again when zle compression is enabled. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 5 +++++ sbin/ggate/ggated/ggated.c | 47 +++++++++++++++++++++++++++++++++++++++------- sbin/ggate/shared/ggate.c | 2 ++ sbin/ggate/shared/ggate.h | 1 + 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 4d0b12f940ee..e70cf553f359 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,10 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + case BIO_DELETE: + g_gate_log(LOG_DEBUG, "DELETE request"); + hdr.gh_cmd = GGATE_CMD_DELETE; + break; case BIO_FLUSH: g_gate_log(LOG_DEBUG, "FLUSH request"); hdr.gh_cmd = GGATE_CMD_FLUSH; @@ -237,6 +241,7 @@ recv_thread(void *arg __unused) if (ggio.gctl_cmd != GGATE_CMD_READ && ggio.gctl_cmd != GGATE_CMD_WRITE && + ggio.gctl_cmd != GGATE_CMD_DELETE && ggio.gctl_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index fa11d8b90d13..c640b5798f69 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -636,6 +636,7 @@ recv_thread(void *arg) * Get header packet. */ req = malloc_waitok(sizeof(*req)); + memset(req, 0, sizeof(*req)); data = g_gate_recv(fd, &req->r_hdr, sizeof(req->r_hdr), MSG_WAITALL); if (data == 0) { @@ -658,6 +659,7 @@ recv_thread(void *arg) */ if (req->r_cmd != GGATE_CMD_READ && req->r_cmd != GGATE_CMD_WRITE && + req->r_cmd != GGATE_CMD_DELETE && req->r_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Request contains invalid command: %d", req->r_cmd); @@ -677,21 +679,16 @@ recv_thread(void *arg) * the client. MAXPHYS is the hard limit in ggatec, * values above it are thus pretty suspicious. */ - if (req->r_length > MAXPHYS) { + if (req->r_length > MAXPHYS && req->r_cmd != GGATE_CMD_DELETE) { g_gate_xlog("Request length above MAXPHYS: %u > %u", (unsigned)req->r_length, MAXPHYS); } /* - * Allocate memory for data, except when flushing. - */ - req->r_data = req->r_cmd != GGATE_CMD_FLUSH ? - malloc_waitok(req->r_length) : NULL; - - /* * Receive data to write for WRITE request. */ if (req->r_cmd == GGATE_CMD_WRITE) { + req->r_data = malloc_waitok(req->r_length); g_gate_log(LOG_DEBUG, "Waiting for %u bytes of data...", req->r_length); data = g_gate_recv(fd, req->r_data, req->r_length, @@ -718,6 +715,34 @@ recv_thread(void *arg) } } +static ssize_t +delete_range(int fd, size_t length, off_t offset) +{ + static char zeros[MAXPHYS]; + size_t written; + + written = 0; + + do + { + int ret; + size_t bytes_left; + size_t chunk_size; + + bytes_left = length - written; + chunk_size = bytes_left > MAXPHYS ? MAXPHYS : bytes_left; + ret = pwrite(fd, zeros, chunk_size, offset + written); + if (ret == -1) + return (written); + written += ret; + } while (written < length); + + g_gate_log(LOG_DEBUG, "Overwritten %u bytes at offset %jd with zeros", + written, (intmax_t)offset); + + return (written); +} + static void * disk_thread(void *arg) { @@ -748,6 +773,7 @@ disk_thread(void *arg) */ assert(req->r_cmd == GGATE_CMD_READ || req->r_cmd == GGATE_CMD_WRITE || + req->r_cmd == GGATE_CMD_DELETE || req->r_cmd == GGATE_CMD_FLUSH); assert(req->r_offset + req->r_length <= (uintmax_t)conn->c_mediasize); assert((req->r_offset % conn->c_sectorsize) == 0); @@ -763,9 +789,16 @@ disk_thread(void *arg) data = 0; switch (req->r_cmd) { case GGATE_CMD_READ: + assert(req->r_data == NULL); + req->r_data = malloc_waitok(req->r_length); data = pread(fd, req->r_data, req->r_length, req->r_offset); break; + case GGATE_CMD_DELETE: + data = delete_range(fd, req->r_length, + req->r_offset); + assert((size_t)data <= req->r_length); + break; case GGATE_CMD_WRITE: data = pwrite(fd, req->r_data, req->r_length, req->r_offset); diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index 05fc25a9afc0..7fe37cc0bc91 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -417,6 +417,8 @@ g_gate_cmd2str(int cmd) return ("GGATE_CMD_READ"); case GGATE_CMD_WRITE: return ("GGATE_CMD_WRITE"); + case GGATE_CMD_DELETE: + return ("GGATE_CMD_DELETE"); case GGATE_CMD_FLUSH: return ("GGATE_CMD_FLUSH"); } diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 36565192a1a8..e5792e3c41e3 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -56,6 +56,7 @@ #define GGATE_CMD_READ 0 #define GGATE_CMD_WRITE 1 #define GGATE_CMD_FLUSH 2 +#define GGATE_CMD_DELETE 3 extern int g_gate_devfd; extern int g_gate_verbose; -- 2.11.0 From dbd036341ac55b6f5efe1d52a00c37abcb5625a6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 29 Apr 2015 10:55:40 +0200 Subject: [PATCH 035/255] ggated send_thread(): Assert that we only send data for read requests Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index c640b5798f69..8e564e136658 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -883,6 +883,7 @@ send_thread(void *arg) g_gate_log(LOG_DEBUG, "Sent hdr packet."); g_gate_swap2h_hdr(&req->r_hdr); if (req->r_data != NULL) { + assert(req->r_cmd == GGATE_CMD_READ); data = g_gate_send(fd, req->r_data, req->r_length, 0); if (data != (ssize_t)req->r_length) { g_gate_xlog("Error while sending data: %s.", -- 2.11.0 From c6de9a6be9f2d5bf66cf2a03fa0b42f1305141e4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:00:04 +0200 Subject: [PATCH 036/255] ggated: Open the listening socket CLOEXEC Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 8e564e136658..49e9c58d67df 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1109,7 +1109,7 @@ main(int argc, char *argv[]) signal(SIGCHLD, SIG_IGN); signal(SIGPIPE, SIG_IGN); - sfd = socket(AF_INET, SOCK_STREAM, 0); + sfd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (sfd == -1) g_gate_xlog("Cannot open stream socket: %s.", strerror(errno)); bzero(&serv, sizeof(serv)); -- 2.11.0 From e36881cc3cc521652949e8bd84fbe93438e7f620 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:31:46 +0200 Subject: [PATCH 037/255] ggated: Fix another socket leak Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 49e9c58d67df..383386b57cef 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -341,6 +341,11 @@ exports_check(struct ggd_export *ex, struct g_gate_cinit *cinit, return (EPERM); } } + if (conn->c_diskfd != -1) { + g_gate_log(LOG_DEBUG, "Requested file %s is already open: %d", + ex->e_path, conn->c_diskfd); + return(0); + } if ((conn->c_flags & GGATE_FLAG_RDONLY) != 0) flags = O_RDONLY; else if ((conn->c_flags & GGATE_FLAG_WRONLY) != 0) -- 2.11.0 From b70145f83eeb896b4e721a6ed50e87504fba34b0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 30 Apr 2015 11:52:06 +0200 Subject: [PATCH 038/255] ggated recv_thread(): In case of read-only files, only accept read commands Accepting write commands etc. is not a security problem because the file descriptor isn't writeable anyway, but accepting requests other than reads could hide client bugs. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 383386b57cef..5ae9d2f7a8f1 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -679,6 +679,12 @@ recv_thread(void *arg) "not fit sector size."); } + if ((conn->c_flags & GGATE_FLAG_RDONLY) != 0 + && req->r_cmd != GGATE_CMD_READ) { + g_gate_xlog("%s request received for read-only file", + g_gate_cmd2str(req->r_cmd)); + } + /* * Limit the amount of memory we allocate on behalf of * the client. MAXPHYS is the hard limit in ggatec, -- 2.11.0 From 5c6918aff43b91d5b9b487017cb5f21396b07fec Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 3 May 2015 14:02:02 +0200 Subject: [PATCH 039/255] ggatec: Add log-to-file support Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 5 ++++- sbin/ggate/shared/ggate.c | 26 +++++++++++++++++++++----- sbin/ggate/shared/ggate.h | 1 + 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index e70cf553f359..ba68977cbded 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -616,7 +616,7 @@ main(int argc, char *argv[]) int ch; char *p; - ch = getopt(argc, argv, "fno:p:q:R:S:s:t:T:u:v"); + ch = getopt(argc, argv, "fl:no:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { @@ -625,6 +625,9 @@ main(int argc, char *argv[]) usage(); force = 1; break; + case 'l': + g_gate_open_log(optarg); + break; case 'n': if (action != CREATE && action != RESCUE) usage(); diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index 7fe37cc0bc91..b5799d6c1620 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -26,6 +26,7 @@ * $FreeBSD$ */ +#define _WITH_DPRINTF #include #include #include @@ -59,13 +60,23 @@ int g_gate_devfd = -1; int g_gate_verbose = 0; +static int g_gate_logfd = -1; +void +g_gate_open_log(const char *logfile) +{ + + g_gate_logfd = open(logfile, O_CREAT | O_WRONLY | O_APPEND, S_IWUSR |S_IRUSR); + if (g_gate_logfd == -1) { + g_gate_xlog("Failed to open %s: %s", logfile, strerror(errno)); + } +} void g_gate_vlog(int priority, const char *message, va_list ap) { - if (g_gate_verbose) { + if (g_gate_verbose || g_gate_logfd != -1) { const char *prefix; switch (priority) { @@ -87,10 +98,15 @@ g_gate_vlog(int priority, const char *message, va_list ap) default: prefix = "unknown"; } - - printf("%s: ", prefix); - vprintf(message, ap); - printf("\n"); + if (g_gate_logfd == -1) { + printf("%s: ", prefix); + vprintf(message, ap); + printf("\n"); + } else if (g_gate_verbose || priority != LOG_DEBUG) { + dprintf(g_gate_logfd, "%s: ", prefix); + vdprintf(g_gate_logfd, message, ap); + dprintf(g_gate_logfd, "\n"); + } } else { if (priority != LOG_DEBUG) vsyslog(priority, message, ap); diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index e5792e3c41e3..51f425dc8cbf 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -95,6 +95,7 @@ struct g_gate_hdr { uint16_t gh_error; /* error value (0 if ok) */ } __packed; +void g_gate_open_log(const char *logfile); void g_gate_vlog(int priority, const char *message, va_list ap); void g_gate_log(int priority, const char *message, ...); void g_gate_xvlog(const char *message, va_list ap) __dead2; -- 2.11.0 From 54253f8536daea8c90d8c9b633ab5eb22326d005 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 30 Apr 2015 13:52:39 +0200 Subject: [PATCH 040/255] ggate[cd]: Add Jail and Capsicum support The capsicum support for ggatec is incomplete and only enabled if the -c flag is used as it currently prevents ggatec from reconnecting which is very inconvenient. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.8 | 8 +++ sbin/ggate/ggatec/ggatec.c | 37 +++++++++--- sbin/ggate/ggated/ggated.c | 5 ++ sbin/ggate/shared/ggate.c | 137 +++++++++++++++++++++++++++++++++++++++++++++ sbin/ggate/shared/ggate.h | 2 + 5 files changed, 182 insertions(+), 7 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.8 b/sbin/ggate/ggatec/ggatec.8 index c50416f4e418..57b6d428efff 100644 --- a/sbin/ggate/ggatec/ggatec.8 +++ b/sbin/ggate/ggatec/ggatec.8 @@ -33,6 +33,7 @@ .Sh SYNOPSIS .Nm .Cm create +.Op Fl c .Op Fl n .Op Fl v .Op Fl o Cm ro | wo | rw @@ -48,6 +49,7 @@ .Ar path .Nm .Cm rescue +.Op Fl c .Op Fl n .Op Fl v .Op Fl o Cm ro | wo | rw @@ -104,6 +106,12 @@ providers. .Pp Available options: .Bl -tag -width ".Fl s Cm ro | wo | rw" +.It Fl c +Enter capsicum sandbox. +Currently this prevents +.Nm ggatec +from reconnecting which is somewhat inconvenient. +The flag will go away once this is fixed. .It Fl f Forcibly destroy .Nm ggate diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ba68977cbded..ad2db8836891 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include "ggate.h" @@ -59,6 +60,8 @@ static enum { UNSET, CREATE, DESTROY, LIST, RESCUE } action = UNSET; static const char *path = NULL; static const char *host = NULL; +static in_addr_t host_ip; +static const char *logfile = NULL; static int unit = G_GATE_UNIT_AUTO; static unsigned flags = 0; static int force = 0; @@ -73,6 +76,7 @@ static int sendfd, recvfd; static uint32_t token; static pthread_t sendtd, recvtd; static int reconnect; +static int drop_capabilities = 0; static void usage(void) @@ -353,7 +357,7 @@ handshake(int dir) */ bzero(&serv, sizeof(serv)); serv.sin_family = AF_INET; - serv.sin_addr.s_addr = g_gate_str2ip(host); + serv.sin_addr.s_addr = host_ip; if (serv.sin_addr.s_addr == INADDR_NONE) { g_gate_log(LOG_DEBUG, "Invalid IP/host name: %s.", host); return (-1); @@ -461,7 +465,7 @@ static void mydaemon(void) { - if (g_gate_verbose > 0) + if (logfile == NULL && g_gate_verbose > 0) return; if (daemon(0, 0) == 0) return; @@ -526,6 +530,10 @@ g_gatec_loop(void) signal(SIGUSR1, signop); for (;;) { g_gatec_start(); + + if (cap_sandboxed()) + g_gate_xlog("Got disconnected while being sandboxed."); + g_gate_log(LOG_NOTICE, "Disconnected [%s %s]. Connecting...", host, path); while (!g_gatec_connect()) { @@ -548,9 +556,6 @@ g_gatec_create(void) if (!g_gatec_connect()) g_gate_xlog("Cannot connect: %s.", strerror(errno)); - /* - * Ok, got both sockets, time to create provider. - */ memset(&ggioc, 0, sizeof(ggioc)); ggioc.gctl_version = G_GATE_VERSION; ggioc.gctl_mediasize = mediasize; @@ -573,6 +578,9 @@ g_gatec_create(void) } unit = ggioc.gctl_unit; + if (drop_capabilities) + g_gate_drop_capabilities(sendfd, recvfd); + mydaemon(); g_gatec_loop(); } @@ -585,6 +593,9 @@ g_gatec_rescue(void) if (!g_gatec_connect()) g_gate_xlog("Cannot connect: %s.", strerror(errno)); + if (drop_capabilities) + g_gate_drop_capabilities(sendfd, recvfd); + ggioc.gctl_version = G_GATE_VERSION; ggioc.gctl_unit = unit; ggioc.gctl_seq = 0; @@ -616,17 +627,21 @@ main(int argc, char *argv[]) int ch; char *p; - ch = getopt(argc, argv, "fl:no:p:q:R:S:s:t:T:u:v"); + ch = getopt(argc, argv, "cfl:no:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { + case 'c': + drop_capabilities = 1; + force = 1; + break; case 'f': if (action != DESTROY) usage(); force = 1; break; case 'l': - g_gate_open_log(optarg); + logfile = optarg; break; case 'n': if (action != CREATE && action != RESCUE) @@ -738,7 +753,11 @@ main(int argc, char *argv[]) g_gate_load_module(); g_gate_open_device(); host = argv[0]; + host_ip = g_gate_str2ip(host); path = argv[1]; + if (logfile != NULL) + g_gate_open_log(logfile); + g_gate_drop_privs("hast", host_ip); g_gatec_create(); break; case DESTROY: @@ -762,7 +781,11 @@ main(int argc, char *argv[]) } g_gate_open_device(); host = argv[0]; + host_ip = g_gate_str2ip(host); path = argv[1]; + if (logfile != NULL) + g_gate_open_log(logfile); + g_gate_drop_privs("hast", host_ip); g_gatec_rescue(); break; case UNSET: diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 5ae9d2f7a8f1..e7dbfbff36b8 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -545,6 +545,11 @@ connection_launch(struct ggd_connection *conn) } g_gate_log(LOG_DEBUG, "Process created [%s].", conn->c_path); + if (getuid() == 0) + g_gate_drop_privs("hast", bindaddr); + + g_gate_drop_capabilities(conn->c_sendfd, conn->c_recvfd); + /* * Create condition variables and mutexes for in-queue and out-queue * synchronization. diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index b5799d6c1620..13277eae4a43 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -53,6 +53,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include "ggate.h" @@ -441,3 +446,135 @@ g_gate_cmd2str(int cmd) return ("unknown (invalid?) GGATE command"); } + +/* + * The functions below are based on drop_privs() from ../../hastd/subr.c + * + * Changes: + * - HAST_USER replaced with ggate_user option + * - pjdlog_* replaced with g_gate_xlog(). + * - Don't fall back to chroot if jailing fails. + */ +#define PJDLOG_VERIFY assert +void +g_gate_drop_privs(const char *ggate_user, in_addr_t jail_address) +{ + char jailhost[32]; + struct jail jailst; + struct passwd *pw; + uid_t ruid, euid, suid; + gid_t rgid, egid, sgid; + gid_t gidset[1]; + struct in_addr jail_ip; + /* + * According to getpwnam(3) we have to clear errno before calling the + * function to be able to distinguish between an error and missing + * entry (with is not treated as error by getpwnam(3)). + */ + errno = 0; + pw = getpwnam(ggate_user); + if (pw == NULL) { + if (errno != 0) { + g_gate_xlog("Unable to find info about '%s' user", + ggate_user); + } else { + g_gate_xlog("'%s' user doesn't exist.", ggate_user); + } + } + + jail_ip.s_addr = jail_address; + + bzero(&jailst, sizeof(jailst)); + jailst.version = JAIL_API_VERSION; + jailst.path = pw->pw_dir; + (void)snprintf(jailhost, sizeof(jailhost), "%s-jail", getprogname()); + jailst.hostname = jailhost; + jailst.jailname = NULL; + jailst.ip4s = 1; + jailst.ip4 = &jail_ip; + jailst.ip6s = 0; + jailst.ip6 = NULL; + if (jail(&jailst) == -1) { + g_gate_xlog("Unable to jail process in directory %s", pw->pw_dir); + } + PJDLOG_VERIFY(chdir("/") == 0); + gidset[0] = pw->pw_gid; + if (setgroups(1, gidset) == -1) { + g_gate_xlog("Unable to set groups to gid %u", + (unsigned int)pw->pw_gid); + } + if (setgid(pw->pw_gid) == -1) { + g_gate_xlog("Unable to set gid to %u", + (unsigned int)pw->pw_gid); + } + if (setuid(pw->pw_uid) == -1) { + g_gate_xlog("Unable to set uid to %u", + (unsigned int)pw->pw_uid); + } + + /* + * Better be sure that everything succeeded. + */ + PJDLOG_VERIFY(getresuid(&ruid, &euid, &suid) == 0); + PJDLOG_VERIFY(ruid == pw->pw_uid); + PJDLOG_VERIFY(euid == pw->pw_uid); + PJDLOG_VERIFY(suid == pw->pw_uid); + PJDLOG_VERIFY(getresgid(&rgid, &egid, &sgid) == 0); + PJDLOG_VERIFY(rgid == pw->pw_gid); + PJDLOG_VERIFY(egid == pw->pw_gid); + PJDLOG_VERIFY(sgid == pw->pw_gid); + PJDLOG_VERIFY(getgroups(0, NULL) == 1); + PJDLOG_VERIFY(getgroups(1, gidset) == 1); + PJDLOG_VERIFY(gidset[0] == pw->pw_gid); + + g_gate_log(LOG_DEBUG, "Privileges successfully dropped using " + "jail+setgid+setuid."); +} + +int +g_gate_drop_capabilities(int sendfd, int recvfd) +{ + cap_rights_t rights; + static const unsigned long ggatecmds[] = { + G_GATE_CMD_START, + G_GATE_CMD_DONE, + G_GATE_CMD_CANCEL, + }; + + if (cap_enter() != 0) { + g_gate_xlog("Failed to sandbox using capsicum"); + } + + cap_rights_init(&rights, CAP_PREAD, CAP_PWRITE); + if (cap_rights_limit(sendfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on sendfd %d", sendfd); + } + if (cap_rights_limit(recvfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on recvfd %d", recvfd); + } + + /* Only the client uses this. */ + if (g_gate_devfd != -1) { + cap_rights_init(&rights, CAP_IOCTL, CAP_PREAD, CAP_PWRITE); + if (cap_rights_limit(g_gate_devfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability rights " + "to CAP_IOCTL on ggate descriptor"); + } + if (cap_ioctls_limit(g_gate_devfd, ggatecmds, + sizeof(ggatecmds) / sizeof(ggatecmds[0])) == -1) { + g_gate_xlog("Unable to limit allowed ggate ioctls"); + } + } + cap_rights_init(&rights, CAP_PWRITE); + if (g_gate_logfd != -1 && + cap_rights_limit(g_gate_logfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on logfd %d", g_gate_logfd); + } + + g_gate_log(LOG_DEBUG, "Entered Capsicum sandbox"); + + return (0); +} diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 51f425dc8cbf..cdc3cbfdbb64 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -115,6 +115,8 @@ void g_gate_list(int unit, int verbose); #endif in_addr_t g_gate_str2ip(const char *str); const char *g_gate_cmd2str(int cmd); +void g_gate_drop_privs(const char *ggate_user, in_addr_t jail_address); +int g_gate_drop_capabilities(int sendfd, int recvfd); /* * g_gate_swap2h_* - functions swap bytes to host byte order (from big endian). -- 2.11.0 From f793eac474cd5b9ac59c763aab77851df8c74577 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 28 Apr 2015 13:02:25 +0200 Subject: [PATCH 041/255] Bump GGATE_VERSION due to FLUSH and DELETE support and various bug fixes Unpatched ggate[cd] versions may cause data corruption so we no longer want to speak to them. Obtained from: ElectroBSD --- sbin/ggate/shared/ggate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index cdc3cbfdbb64..e764665ebb85 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -40,7 +40,7 @@ #define G_GATE_TIMEOUT 0 #define GGATE_MAGIC "GEOM_GATE " -#define GGATE_VERSION 0 +#define GGATE_VERSION 1 #define GGATE_FLAG_RDONLY 0x0001 #define GGATE_FLAG_WRONLY 0x0002 -- 2.11.0 From 94c43887fba4bf5f44969e18f4fb8a66528c0fba Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 9 Aug 2015 15:20:48 +0200 Subject: [PATCH 042/255] Use dedicated users for ggatec and ggated Obtained from: ElectroBSD --- etc/group | 2 ++ etc/master.passwd | 2 ++ sbin/ggate/ggatec/ggatec.c | 2 +- sbin/ggate/ggated/ggated.c | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/group b/etc/group index c20f1d93dd3c..3136230b0790 100644 --- a/etc/group +++ b/etc/group @@ -31,5 +31,7 @@ audit:*:77: www:*:80: _ypldap:*:160: hast:*:845: +ggatec:*:846: +ggated:*:847: nogroup:*:65533: nobody:*:65534: diff --git a/etc/master.passwd b/etc/master.passwd index 4b0f3645cd3d..689b64e78473 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -24,4 +24,6 @@ auditdistd:*:78:77::0:0:Auditdistd unprivileged user:/var/empty:/usr/sbin/nologi www:*:80:80::0:0:World Wide Web Owner:/nonexistent:/usr/sbin/nologin _ypldap:*:160:160::0:0:YP LDAP unprivileged user:/var/empty:/usr/sbin/nologin hast:*:845:845::0:0:HAST unprivileged user:/var/empty:/usr/sbin/nologin +ggatec:*:846:846::0:0:ggatec unprivileged user:/var/empty:/usr/sbin/nologin +ggated:*:847:847::0:0:ggated unprivileged user:/var/empty:/usr/sbin/nologin nobody:*:65534:65534::0:0:Unprivileged user:/nonexistent:/usr/sbin/nologin diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ad2db8836891..cd33e1238ac5 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -757,7 +757,7 @@ main(int argc, char *argv[]) path = argv[1]; if (logfile != NULL) g_gate_open_log(logfile); - g_gate_drop_privs("hast", host_ip); + g_gate_drop_privs("ggatec", host_ip); g_gatec_create(); break; case DESTROY: diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index e7dbfbff36b8..439725314aab 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -546,7 +546,7 @@ connection_launch(struct ggd_connection *conn) g_gate_log(LOG_DEBUG, "Process created [%s].", conn->c_path); if (getuid() == 0) - g_gate_drop_privs("hast", bindaddr); + g_gate_drop_privs("ggated", bindaddr); g_gate_drop_capabilities(conn->c_sendfd, conn->c_recvfd); -- 2.11.0 From 68f702f7cf2edd6200f39ea35f936a9fa8c18c7c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Jan 2015 18:50:18 +0100 Subject: [PATCH 043/255] bge(4): Default to disallowing ASF It causes watchdog timeouts and undiagnosed permanent unresponsivenes on at least the 'CHIP ID 0x05784100; ASIC REV 0x5784; CHIP REV 0x57841;' in the evo-iv08 DL120 G6 I'm using for testing. I don't have time to debug the underlying cause right now and users who actually want ASF and have systems where it works can always enable it through loader.conf. Obtained from: ElectroBSD --- share/man/man4/bge.4 | 4 ++-- sys/dev/bge/if_bge.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/share/man/man4/bge.4 b/share/man/man4/bge.4 index 474b48fa5ff1..060d22d76658 100644 --- a/share/man/man4/bge.4 +++ b/share/man/man4/bge.4 @@ -31,7 +31,7 @@ .\" .\" $FreeBSD$ .\" -.Dd January 19, 2012 +.Dd January 19, 2015 .Dt BGE 4 .Os .Sh NAME @@ -196,7 +196,7 @@ prompt before booting the kernel, or stored in .It Va hw.bge.allow_asf Allow the ASF feature for cooperating with IPMI. Can cause system lockup problems on a small number of systems. -Enabled by default. +Disabled by default. .It Va dev.bge.%d.msi Non-zero value enables MSI support on the Ethernet hardware. The default value is 1. diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c index 929bedc5ee0a..b89b8fb93af8 100644 --- a/sys/dev/bge/if_bge.c +++ b/sys/dev/bge/if_bge.c @@ -543,7 +543,7 @@ static devclass_t bge_devclass; DRIVER_MODULE(bge, pci, bge_driver, bge_devclass, 0, 0); DRIVER_MODULE(miibus, bge, miibus_driver, miibus_devclass, 0, 0); -static int bge_allow_asf = 1; +static int bge_allow_asf = 0; static SYSCTL_NODE(_hw, OID_AUTO, bge, CTLFLAG_RD, 0, "BGE driver parameters"); SYSCTL_INT(_hw_bge, OID_AUTO, allow_asf, CTLFLAG_RDTUN, &bge_allow_asf, 0, -- 2.11.0 From 81df7d2aab2598a48bdacb27a3db9b0adaed2393 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Jan 2015 21:25:47 +0100 Subject: [PATCH 044/255] Assign random IP id values by default so users don't have to clown around with the sysctl themselves Obtained from: ElectroBSD --- sys/netinet/ip_id.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/netinet/ip_id.c b/sys/netinet/ip_id.c index 97d5851b349f..8651c6404730 100644 --- a/sys/netinet/ip_id.c +++ b/sys/netinet/ip_id.c @@ -137,8 +137,8 @@ static void ipid_sysuninit(void); SYSCTL_DECL(_net_inet_ip); SYSCTL_PROC(_net_inet_ip, OID_AUTO, random_id, CTLTYPE_INT | CTLFLAG_VNET | CTLFLAG_RW, - &VNET_NAME(ip_do_randomid), 0, sysctl_ip_randomid, "IU", - "Assign random ip_id values"); + &VNET_NAME(ip_do_randomid), 1, sysctl_ip_randomid, "IU", + "Assign random ip_id values. Important for Tor relays and a good idea in general."); SYSCTL_INT(_net_inet_ip, OID_AUTO, rfc6864, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip_rfc6864), 0, "Use constant IP ID for atomic datagrams"); -- 2.11.0 From a84718324d45c15f4db564d727b0c30e88067e83 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 11 Feb 2015 12:19:24 +0100 Subject: [PATCH 045/255] Add vendor copyright ... after putting on my robe and "police educational technican" hat. This commit is optional. Feel free to import any other ElectroBSD commit without including this one. Obtained from: ElectroBSD --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index e9c19881f284..ab9093d76694 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -30,7 +30,7 @@ /* Add a FreeBSD vendor copyright here */ #define COPYRIGHT_Vendor \ - "" + "Copyright (c) 2010-2015 Fabian Keil - IT-Beratung und Polizei-Erziehung\n" /* FreeBSD */ #define COPYRIGHT_FreeBSD \ -- 2.11.0 From 38e7a83ce0625cb7e85a3a6e2dad06cf7c2c25bd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 13 Feb 2015 19:37:33 +0100 Subject: [PATCH 046/255] OpenZFS: fix a comment typo Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c index 2a3939f4dec0..df53edad206c 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c @@ -1555,7 +1555,7 @@ int zfs_frag_table[FRAGMENTATION_TABLE_SIZE] = { }; /* - * Calclate the metaslab's fragmentation metric. A return value + * Calculate the metaslab's fragmentation metric. A return value * of ZFS_FRAG_INVALID means that the metaslab has not been upgraded and does * not support this metric. Otherwise, the return value should be in the * range [0, 100]. -- 2.11.0 From 1b44739835d9906079f2792f71df867ca142de5c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 Feb 2015 11:38:09 +0100 Subject: [PATCH 047/255] newvers.sh: Set TYPE to ElectroBSD Obtained from: ElectroBSD --- sys/conf/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 sys/conf/newvers.sh diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh old mode 100644 new mode 100755 index a1539532eb01..95a55cb8608a --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -30,7 +30,7 @@ # @(#)newvers.sh 8.1 (Berkeley) 4/20/94 # $FreeBSD$ -TYPE="FreeBSD" +TYPE="ElectroBSD" REVISION="11.0" BRANCH="STABLE" if [ -n "${BRANCH_OVERRIDE}" ]; then -- 2.11.0 From 8f9cb64751212e9feb77805d5584723cdc288f2a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 Jan 2016 13:45:32 +0100 Subject: [PATCH 048/255] mdocml: Change OS name used in man page headers to ElectroBSD Obtained from: ElectroBSD --- contrib/mdocml/msec.in | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/mdocml/msec.in b/contrib/mdocml/msec.in index 86d0dd8486f2..09f6f362c685 100644 --- a/contrib/mdocml/msec.in +++ b/contrib/mdocml/msec.in @@ -22,16 +22,16 @@ * Be sure to escape strings. */ -LINE("1", "FreeBSD General Commands Manual") -LINE("2", "FreeBSD System Calls Manual") -LINE("3", "FreeBSD Library Functions Manual") +LINE("1", "ElectroBSD General Commands Manual") +LINE("2", "ElectroBSD System Calls Manual") +LINE("3", "ElectroBSD Library Functions Manual") LINE("3p", "Perl Library Functions Manual") -LINE("4", "FreeBSD Kernel Interfaces Manual") -LINE("5", "FreeBSD File Formats Manual") -LINE("6", "FreeBSD Games Manual") -LINE("7", "FreeBSD Miscellaneous Information Manual") -LINE("8", "FreeBSD System Manager\'s Manual") -LINE("9", "FreeBSD Kernel Developer\'s Manual") +LINE("4", "ElectroBSD Kernel Interfaces Manual") +LINE("5", "ElectroBSD File Formats Manual") +LINE("6", "ElectroBSD Games Manual") +LINE("7", "ElectroBSD Miscellaneous Information Manual") +LINE("8", "ElectroBSD System Manager\'s Manual") +LINE("9", "ElectroBSD Kernel Developer\'s Manual") LINE("X11", "X11 Developer\'s Manual") LINE("X11R6", "X11 Developer\'s Manual") LINE("unass", "Unassociated") -- 2.11.0 From 069b63e83c02b00e1f49f5f29c43ca49a1052869 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 Jan 2016 20:24:00 +0100 Subject: [PATCH 049/255] clang: Set CLANG_VENDOR to ElectroBSD Obtained from: ElectroBSD --- lib/clang/include/clang/Basic/Version.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/clang/include/clang/Basic/Version.inc b/lib/clang/include/clang/Basic/Version.inc index b9d37e10f845..fa5700f89a58 100644 --- a/lib/clang/include/clang/Basic/Version.inc +++ b/lib/clang/include/clang/Basic/Version.inc @@ -5,6 +5,6 @@ #define CLANG_VERSION_MINOR 9 #define CLANG_VERSION_PATCHLEVEL 1 -#define CLANG_VENDOR "FreeBSD " +#define CLANG_VENDOR "ElectroBSD " #define SVN_REVISION "289601" -- 2.11.0 From c22e9be51aa84470525d6dd6ed3eeb76b5e3bd15 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 24 Feb 2015 19:35:03 +0100 Subject: [PATCH 050/255] Let rc.d/motd work with unames other than FreeBSD Obtained from: ElectroBSD --- etc/rc.d/motd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/rc.d/motd b/etc/rc.d/motd index acb376723e80..11d2257aa9c0 100755 --- a/etc/rc.d/motd +++ b/etc/rc.d/motd @@ -35,7 +35,7 @@ motd_start() T=`mktemp -t motd` uname -v | sed -e 's,^\([^#]*\) #\(.* [1-2][0-9][0-9][0-9]\).*/\([^\]*\) $,\1 (\3) #\2,' > ${T} - awk '{if (NR == 1) {if ($1 == "FreeBSD") {next} else {print "\n"$0}} else {print}}' < /etc/motd >> ${T} + awk '{if (NR == 1) {if ($1 == "'"$(uname)"'") {next} else {print "\n"$0}} else {print}}' < /etc/motd >> ${T} cmp -s $T /etc/motd || { cp $T /etc/motd -- 2.11.0 From 9f240ef65a392fff5d15e871f525ce3eac23fe36 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 Mar 2015 12:07:50 +0100 Subject: [PATCH 051/255] swapon: Accept the "late" option for .eli devices Using the late option makes it possible to additionally use the consumer device as (unencrypted!) dump device without making it unavailable for rc.d/savecore (which runs between rc.d/swap and rc.d/swaplate). Obtained from: ElectroBSD Submitted in: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198598 Submitted at: 2015-03-15 11:37 UTC --- sbin/swapon/swapon.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbin/swapon/swapon.c b/sbin/swapon/swapon.c index 052fbc734afb..dfc4951c8137 100644 --- a/sbin/swapon/swapon.c +++ b/sbin/swapon/swapon.c @@ -377,7 +377,8 @@ swap_on_geli_args(const char *mntops) } } else if ((p = strstr(token, "notrim")) == token) { Tflag = " -T "; - } else if (strcmp(token, "sw") != 0) { + } else if (strcmp(token, "sw") != 0 && + strcmp(token, "late") != 0) { warnx("Invalid option: %s", token); free(ops); return (NULL); -- 2.11.0 From 3583f288a1702feeceff71df94ab9f1eaf4063b0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 18 Apr 2015 19:38:07 +0200 Subject: [PATCH 052/255] gmountver.8: Note that GEOM mount verification has dangerous bugs Obtained from: ElectroBSD --- sbin/geom/class/mountver/gmountver.8 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sbin/geom/class/mountver/gmountver.8 b/sbin/geom/class/mountver/gmountver.8 index 4c27a652b0b0..ddb9e2b2a73f 100644 --- a/sbin/geom/class/mountver/gmountver.8 +++ b/sbin/geom/class/mountver/gmountver.8 @@ -61,6 +61,8 @@ got disconnected - it queues all the I/O requests and waits for the provider to reappear. When that happens, it attaches to it and sends the queued requests. .Pp +At least that's the theory, please note the BUGS section. +.Pp The first argument to .Nm indicates an action to be performed: @@ -119,6 +121,13 @@ If set to 0, .Nm will reattach to the device even if the device reports a different disk ID. .El +.Sh BUGS +The mount verification GEOM class can stall all the disk I/O instead +of just the device it is configured for. +The disk identification check currently has to be turned off for the class +to actually attach reappearing providers. +Obviously this is dangerous. +.El .Sh EXIT STATUS Exit status is 0 on success, and 1 if the command fails. .Sh SEE ALSO -- 2.11.0 From 5d312320b0e412e119462a8f8f736ca2da80137f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 19 Apr 2015 22:58:49 +0200 Subject: [PATCH 053/255] Default to listening to 127.0.0.1 only Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 439725314aab..858dc0783fb1 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1055,7 +1055,7 @@ main(int argc, char *argv[]) int ch, sfd, tmpsfd; unsigned port; - bindaddr = htonl(INADDR_ANY); + bindaddr = g_gate_str2ip("127.0.0.1"); port = G_GATE_PORT; while ((ch = getopt(argc, argv, "a:hnp:F:R:S:v")) != -1) { switch (ch) { -- 2.11.0 From 611e87839a7090083da563d366d101e6d0bf8af7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 30 Mar 2015 15:24:05 +0200 Subject: [PATCH 054/255] ggatel: Optionally retry in case of failed reads and writes ... after waiting five seconds and reopening the device. This allows to use ggatel as a workaround against USB flakiness which can result in device disconnects that are extremely annoying if ZFS on geli is involved. To prevent data loss if multiple device disappear at the same time and reappear with different names, the disk ident is checked to confirm that the device is the expected one. As a side-effect retrying will not work for files (which have no disk ident). Obtained from: ElectroBSD --- sbin/ggate/ggatel/ggatel.c | 63 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/sbin/ggate/ggatel/ggatel.c b/sbin/ggate/ggatel/ggatel.c index 0b89df091d2a..864f3ae40d70 100644 --- a/sbin/ggate/ggatel/ggatel.c +++ b/sbin/ggate/ggatel/ggatel.c @@ -53,6 +53,7 @@ static const char *path = NULL; static int unit = G_GATE_UNIT_AUTO; static unsigned flags = 0; static int force = 0; +static unsigned retries = 0; static unsigned sectorsize = 0; static unsigned timeout = G_GATE_TIMEOUT; @@ -60,10 +61,10 @@ static void usage(void) { - fprintf(stderr, "usage: %s create [-v] [-o ] " + fprintf(stderr, "usage: %s create [-v] [-o ] [-r ] " "[-s sectorsize] [-t timeout] [-u unit] \n", getprogname()); - fprintf(stderr, " %s rescue [-v] [-o ] <-u unit> " - "\n", getprogname()); + fprintf(stderr, " %s rescue [-v] [-o ] [-r ] " + "<-u unit> \n", getprogname()); fprintf(stderr, " %s destroy [-f] <-u unit>\n", getprogname()); fprintf(stderr, " %s list [-v] [-u unit]\n", getprogname()); exit(EXIT_FAILURE); @@ -85,6 +86,7 @@ g_gatel_serve(int fd) { struct g_gate_ctl_io ggio; size_t bsize; + char ident[DISK_IDENT_SIZE]; if (g_gate_verbose == 0) { if (daemon(0, 0) == -1) { @@ -97,8 +99,15 @@ g_gatel_serve(int fd) ggio.gctl_unit = unit; bsize = sectorsize; ggio.gctl_data = malloc(bsize); + + errno = 0; + if (retries && ioctl(fd, DIOCGIDENT, ident) != 0) { + g_gate_xlog("Failed to get disk ident for %s: %s", path, + strerror(errno)); + } for (;;) { int error; + int retries_left; once_again: ggio.gctl_length = bsize; ggio.gctl_error = 0; @@ -130,6 +139,8 @@ once_again: strerror(error)); } + retries_left = retries; +retry_request: error = 0; switch (ggio.gctl_cmd) { case BIO_READ: @@ -145,6 +156,10 @@ once_again: if (pread(fd, ggio.gctl_data, ggio.gctl_length, ggio.gctl_offset) == -1) { error = errno; + g_gate_log(LOG_ERR, "Failed to read %d" + " bytes from %s: %s", + ggio.gctl_length, path, + strerror(error)); } } break; @@ -153,12 +168,44 @@ once_again: if (pwrite(fd, ggio.gctl_data, ggio.gctl_length, ggio.gctl_offset) == -1) { error = errno; + g_gate_log(LOG_ERR, "Failed to write %d bytes" + " to %s: %s", ggio.gctl_length, path, + strerror(error)); } break; default: error = EOPNOTSUPP; } + if (error && error != EOPNOTSUPP) { + if (retries_left > 0) { + char ident_new[DISK_IDENT_SIZE]; + + close(fd); + retries_left--; + sleep(5); + fd = open(path, g_gate_openflags(flags) | + O_DIRECT | O_FSYNC); + if (fd == -1) { + err(EXIT_FAILURE, "Cannot open %s", + path); + } + if (ioctl(fd, DIOCGIDENT, ident_new) != 0) { + g_gate_xlog("Failed to get disk ", + "ident for %s: %s", path, + strerror(errno)); + } + if (strcmp(ident, ident_new) != 0) { + g_gate_xlog("Disk ident for %s " + "changed from %s to %s. Reuse " + "could cause data loss.", path, + ident, ident_new); + } + g_gate_log(LOG_ERR, "Retrying after reopening " + "%s (%s)", path, ident); + goto retry_request; + } + } ggio.gctl_error = error; g_gate_ioctl(G_GATE_CMD_DONE, &ggio); } @@ -230,7 +277,7 @@ main(int argc, char *argv[]) for (;;) { int ch; - ch = getopt(argc, argv, "fo:s:t:u:v"); + ch = getopt(argc, argv, "fo:r:s:t:u:v"); if (ch == -1) break; switch (ch) { @@ -253,6 +300,14 @@ main(int argc, char *argv[]) "Invalid argument for '-o' option."); } break; + case 'r': + if (action != CREATE && action != RESCUE) + usage(); + errno = 0; + retries = strtoul(optarg, NULL, 10); + if (retries == 0 && errno != 0) + errx(EXIT_FAILURE, "Invalid retry count."); + break; case 's': if (action != CREATE) usage(); -- 2.11.0 From f79e0e3a3866325176610081649f889d33731c0b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 26 Apr 2015 17:54:14 +0200 Subject: [PATCH 055/255] ggatel.8: Document the shiny new -r option Obtained from: ElectroBSD --- sbin/ggate/ggatel/ggatel.8 | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/sbin/ggate/ggatel/ggatel.8 b/sbin/ggate/ggatel/ggatel.8 index f2eea7cf3fd5..7612b7509bd1 100644 --- a/sbin/ggate/ggatel/ggatel.8 +++ b/sbin/ggate/ggatel/ggatel.8 @@ -35,6 +35,7 @@ .Cm create .Op Fl v .Op Fl o Cm ro | wo | rw +.Op Fl r Ar retries .Op Fl s Ar sectorsize .Op Fl t Ar timeout .Op Fl u Ar unit @@ -51,6 +52,7 @@ .Cm rescue .Op Fl v .Op Fl o Cm ro | wo | rw +.Op Fl r Ar retries .Fl u Ar unit .Ar path .Sh DESCRIPTION @@ -102,6 +104,19 @@ or read-write .Pq Cm rw . Default is .Cm rw . +.It Fl r Ar retries +Number of times a failed request should be retried before forwarding +the error to the kernel. +Between retries, +.Nm ggatel +waits for five seconds and reopens the device in case it temporarily +disappeared. +The reopened device is only used if the disk identification did not +change. +This option is useful when using unreliable USB devices as geli +consumer (as long as the device loss does not cause the USB +stack to deadlock). +By default failed requests are not retried. .It Fl s Ar sectorsize Sector size for .Nm ggate @@ -144,6 +159,52 @@ umount /secret gbde detach ggate5 ggatel destroy -u 5 .Ed + +Scrub a pool on an USB device that occasionally disappears: +.Bd -literal -offset indent +$ glabel list da0 +Geom name: da0 +Providers: +1. Name: label/extreme + Mediasize: 4023385600 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + secoffset: 0 + offset: 0 + seclength: 7858175 + length: 4023385600 + index: 0 +Consumers: +1. Name: da0 + Mediasize: 4023386112 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + +$ sudo ggatel create -r 2 /dev/da0 +ggate0 +$ glabel list da0 +glabel: No such geom: da0. +glabel list ggate0 +Geom name: ggate0 +Providers: +1. Name: label/extreme + Mediasize: 4023385600 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + secoffset: 0 + offset: 0 + seclength: 7858175 + length: 4023385600 + index: 0 +Consumers: +1. Name: ggate0 + Mediasize: 4023386112 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + +$ zogftw import extreme +$ sudo zpool scrub extreme +.Ed .Sh SEE ALSO .Xr geom 4 , .Xr gbde 8 , -- 2.11.0 From 5775527f36bfdf0bfca148840635fd38edebce71 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:48:20 +0200 Subject: [PATCH 056/255] g_multipath: Add sysctl to disable tasting Obtained from: ElectroBSD --- sys/geom/multipath/g_multipath.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sys/geom/multipath/g_multipath.c b/sys/geom/multipath/g_multipath.c index b461747bf356..b97d515a4b8e 100644 --- a/sys/geom/multipath/g_multipath.c +++ b/sys/geom/multipath/g_multipath.c @@ -58,6 +58,9 @@ SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW, static u_int g_multipath_exclusive = 1; SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW, &g_multipath_exclusive, 0, "Exclusively open providers"); +static u_int g_multipath_enable_tasting = 1; +SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, taste, CTLFLAG_RW, + &g_multipath_enable_tasting, 0, "Enable multipath tasting. May cause conflicts."); static enum { GKT_NIL, @@ -798,6 +801,9 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) g_topology_assert(); + if (g_multipath_enable_tasting == 0) + return (NULL); + gp = g_new_geomf(mp, "multipath:taste"); gp->start = g_multipath_start; gp->access = g_multipath_access; -- 2.11.0 From 996589adde81b1b7587260881af33eea281fe13b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 May 2015 14:32:31 +0200 Subject: [PATCH 057/255] motd: Customize for ElectroBSD Obtained from: ElectroBSD --- etc/motd | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/etc/motd b/etc/motd index 4dc41a744c31..3069d8ed4b20 100644 --- a/etc/motd +++ b/etc/motd @@ -1,21 +1,8 @@ -FreeBSD ?.?.? (UNKNOWN) +ElectroBSD ?.?.? (UNKNOWN) -Welcome to FreeBSD! +Welcome to ElectroBSD! -Release Notes, Errata: https://www.FreeBSD.org/releases/ -Security Advisories: https://www.FreeBSD.org/security/ -FreeBSD Handbook: https://www.FreeBSD.org/handbook/ -FreeBSD FAQ: https://www.FreeBSD.org/faq/ -Questions List: https://lists.FreeBSD.org/mailman/listinfo/freebsd-questions/ -FreeBSD Forums: https://forums.FreeBSD.org/ - -Documents installed with the system are in the /usr/local/share/doc/freebsd/ -directory, or can be installed later with: pkg install en-freebsd-doc -For other languages, replace "en" with a language code like de or fr. - -Show the version of FreeBSD installed: freebsd-version ; uname -a -Please include that output and any error messages when posting questions. -Introduction to manual pages: man man -FreeBSD directory layout: man hier +For details see: +https://www.ElectroBSD.org/ Edit /etc/motd to change this login announcement. -- 2.11.0 From fea2c9b0d70316d08619464001b8755bb3dd1d9c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 14:18:43 +0200 Subject: [PATCH 058/255] release: Load usb modules through loader.conf so 'memstick' works with ELECTRO_BEER Obtained from: ElectroBSD --- release/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/Makefile b/release/Makefile index b1dfa82ff8fd..60ffead48a54 100644 --- a/release/Makefile +++ b/release/Makefile @@ -239,6 +239,9 @@ dvd: packagesystem echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf + for module in usb usb_quirk ehci umass; do \ + echo $${module}_load=\"YES\" >> ${.TARGET}/boot/loader.conf + done cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.11.0 From 9a3da0acfcb554989d79c4bac0cd45646de6a3ee Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:13:51 +0200 Subject: [PATCH 059/255] brand-fbsd.4th: Change OS name in banner to ElectroBSD While at it, suggest to resist unlawful police activities (German). Obtained from: ElectroBSD --- sys/boot/forth/brand-fbsd.4th | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sys/boot/forth/brand-fbsd.4th b/sys/boot/forth/brand-fbsd.4th index 9cd017f84a3f..03392cab8403 100644 --- a/sys/boot/forth/brand-fbsd.4th +++ b/sys/boot/forth/brand-fbsd.4th @@ -32,15 +32,17 @@ 1+ \ increase y for next time we're called ; -: brand ( x y -- ) \ "FreeBSD" [wide] logo in B/W (7 rows x 42 columns) +: brand ( x y -- ) \ "ElectroBSD" [wide] logo in B/W - s" ______ ____ _____ _____ " brand+ - s" | ____| | _ \ / ____| __ \ " brand+ - s" | |___ _ __ ___ ___ | |_) | (___ | | | |" brand+ - s" | ___| '__/ _ \/ _ \| _ < \___ \| | | |" brand+ - s" | | | | | __/ __/| |_) |____) | |__| |" brand+ - s" | | | | | | || | | |" brand+ - s" |_| |_| \___|\___||____/|_____/|_____/ " brand+ + s" ______ _ _ ____ _____ _____" brand+ + s" | ____| | | | | _ \ / ____| __ \" brand+ + s" | |__ | | ___ ___| |_ _ __ ___ | |_) | (___ | | | |" brand+ + s" | __| | |/ _ \/ __| __| '__/ _ \| _ < \___ \| | | |" brand+ + s" | |____| | __/ (__| |_| | | (_) | |_) |____) | |__| |" brand+ + s" |______|_|\___|\___|\__|_| \___/|____/|_____/|_____/" brand+ + s" Polizei-Willkuer in Deinem Land? Das erfordert Widerstand!" brand+ + s" Zu Risiken und Nebenwirkungen fragen Sie Ihren Anwalt oder" brand+ + s" die Rote Hilfe." brand+ 2drop ; -- 2.11.0 From 88a047b4eeca7c7dec8aa68a43248c2a6f323538 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:25:56 +0200 Subject: [PATCH 060/255] beastie.4th: Think of the children and default to showing beastie instead of the sex toy Obtained from: ElectroBSD --- sys/boot/forth/beastie.4th | 4 ++-- sys/boot/forth/beastie.4th.8 | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sys/boot/forth/beastie.4th b/sys/boot/forth/beastie.4th index 752cce22a4ff..6a28605bcefd 100644 --- a/sys/boot/forth/beastie.4th +++ b/sys/boot/forth/beastie.4th @@ -64,9 +64,9 @@ variable logoY s" loader_logo" getenv dup -1 = over 0= or if dup 0= if 2drop else drop then \ getenv result unused loader_color? if - s" try-include /boot/logo-orb.4th" + s" try-include /boot/logo-beastie.4th" else - s" try-include /boot/logo-orbbw.4th" + s" try-include /boot/logo-beastiebw.4th" then else 2drop ( c-addr/u -- ) \ getenv result unused diff --git a/sys/boot/forth/beastie.4th.8 b/sys/boot/forth/beastie.4th.8 index 9f77d5db2977..3ea13c1830ea 100644 --- a/sys/boot/forth/beastie.4th.8 +++ b/sys/boot/forth/beastie.4th.8 @@ -106,10 +106,9 @@ The environment variables that effect its behavior are: Selects the desired logo in the beastie boot menu. Possible values are: .Dq Li fbsdbw , .Dq Li beastie , -.Dq Li beastiebw , +.Dq Li beastiebw (default) , .Dq Li orb , -.Dq Li orbbw -(default), and +.Dq Li orbbw , and .Dq Li none . .It Va loader_logo_x Sets the desired column position of the logo. Default is 46. -- 2.11.0 From 5b2b876843b2ac485f4db6bc9f3b48b7984bf19f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:39:56 +0200 Subject: [PATCH 061/255] menu.4th: Reduce visual noise by ditching the welcome text Obtained from: ElectroBSD --- sys/boot/forth/menu.4th | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/forth/menu.4th b/sys/boot/forth/menu.4th index e3fe0f7d776e..18d1671db6d2 100644 --- a/sys/boot/forth/menu.4th +++ b/sys/boot/forth/menu.4th @@ -470,7 +470,7 @@ also menu-infrastructure definitions \ Print the frame caption at (x,y) s" loader_menu_title" getenv dup -1 = if - drop s" Welcome to FreeBSD" + drop s" " then TRUE ( use default alignment ) s" loader_menu_title_align" getenv dup -1 <> if -- 2.11.0 From e536824a4ea82e9baf41fc339cf9d81168f9c9d5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 17:40:22 +0200 Subject: [PATCH 062/255] logo-beastiebw: Change the fork to a toilet brush Obtained from: ElectroBSD --- sys/boot/forth/logo-beastiebw.4th | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sys/boot/forth/logo-beastiebw.4th b/sys/boot/forth/logo-beastiebw.4th index 197099cda0bc..bce43f5b9d52 100644 --- a/sys/boot/forth/logo-beastiebw.4th +++ b/sys/boot/forth/logo-beastiebw.4th @@ -33,27 +33,27 @@ 1+ \ increase y for next time we're called ; -: logo ( x y -- ) \ B/W BSD mascot (19 rows x 34 columns) +: logo ( x y -- ) \ B/W BSD mascot with toilet brush - s" , ," logo+ - s" /( )`" logo+ - s" \ \___ / |" logo+ - s" /- _ `-/ '" logo+ - s" (/\/ \ \ /\" logo+ - s" / / | ` \" logo+ - s" O O ) / |" logo+ - s" `-^--'`< '" logo+ - s" (_.) _ ) /" logo+ - s" `.___/` /" logo+ - s" `-----' /" logo+ - s" <----. __ / __ \" logo+ - s" <----|====O)))==) \) /====|" logo+ - s" <----' `--' `.__,' \" logo+ - s" | |" logo+ - s" \ / /\" logo+ - s" ______( (_ / \______/" logo+ - s" ,' ,-----' |" logo+ - s" `--{__________)" logo+ + s" , ," logo+ + s" /( )`" logo+ + s" \ \___ / |" logo+ + s" /- _ `-/ '" logo+ + s" (/\/ \ \ /\" logo+ + s" / / | ` \" logo+ + s" O O ) / |" logo+ + s" `-^--'`< '" logo+ + s" (_.) _ ) /" logo+ + s" `.___/` /" logo+ + s" `-----' /" logo+ + s" ###### __ / __ \" logo+ + s" ######===O)))==) \) /====|" logo+ + s" ###### `--' `.__,' \" logo+ + s" | |" logo+ + s" \ / /\" logo+ + s" ______( (_ / \___/" logo+ + s" ,' ,-----' |" logo+ + s" `--{__________)" logo+ 2drop ; -- 2.11.0 From beff816bbdc2261c8a6b9d7d8eb3f8bc9cdd10bc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 18:19:19 +0200 Subject: [PATCH 063/255] logo-beastie.4th: Replace fork with toilet brush Obtained from: ElectroBSD --- sys/boot/forth/logo-beastie.4th | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sys/boot/forth/logo-beastie.4th b/sys/boot/forth/logo-beastie.4th index 671eb5e496b2..8441c9ddafbd 100644 --- a/sys/boot/forth/logo-beastie.4th +++ b/sys/boot/forth/logo-beastie.4th @@ -35,27 +35,27 @@ 1+ \ increase y for next time we're called ; -: logo ( x y -- ) \ color BSD mascot (19 rows x 34 columns) +: logo ( x y -- ) \ color BSD mascot with toilet brush - s" @[31m, ," logo+ - s" /( )`" logo+ - s" \ \___ / |" logo+ - s" /- @[m_@[31m `-/ '" logo+ - s" (@[m/\/ \@[31m \ /\" logo+ - s" @[m/ / |@[31m ` \" logo+ - s" @[34mO O @[m) @[31m/ |" logo+ - s" @[m`-^--'@[31m`< '" logo+ - s" (_.) _ ) /" logo+ - s" `.___/` /" logo+ - s" `-----' /" logo+ - s" @[33m<----.@[31m __ / __ \" logo+ - s" @[33m<----|====@[31mO)))@[33m==@[31m) \) /@[33m====|" logo+ - s" @[33m<----'@[31m `--' `.__,' \" logo+ - s" | |" logo+ - s" \ / /\" logo+ - s" @[36m______@[31m( (_ / \______/" logo+ - s" @[36m,' ,-----' |" logo+ - s" `--{__________)@[m" logo+ + s" @[31m, ," logo+ + s" /( )`" logo+ + s" \ \___ / |" logo+ + s" /- @[m_@[31m `-/ '" logo+ + s" (@[m/\/ \@[31m \ /\" logo+ + s" @[m/ / |@[31m ` \" logo+ + s" @[34mO O @[m) @[31m/ |" logo+ + s" @[m`-^--'@[31m`< '" logo+ + s" (_.) _ ) /" logo+ + s" `.___/` /" logo+ + s" `-----' /" logo+ + s" @[37m######@[31m __ / __ \" logo+ + s" @[37m######====@[31mO)))@[37m==@[31m) \) /@[37m====|" logo+ + s" @[37m######@[31m `--' `.__,' \" logo+ + s" | |" logo+ + s" \ / /\" logo+ + s" @[36m______@[31m( (_ / \_____/" logo+ + s" @[36m,' ,-----' |" logo+ + s" `--{__________)@[m" logo+ 2drop ; -- 2.11.0 From 3133eeed63c9acfafb263bd51cc815c4fc6576bc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 13:24:26 +0200 Subject: [PATCH 064/255] newsyslog.conf: Do not give world read permissions and don't compress stuff Obtained from: ElectroBSD --- etc/newsyslog.conf | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/etc/newsyslog.conf b/etc/newsyslog.conf index ab595cac61db..a40949390c1e 100644 --- a/etc/newsyslog.conf +++ b/etc/newsyslog.conf @@ -10,34 +10,28 @@ # # The 'flags' field is one or more of the letters: BCDGJNUXZ or a '-'. # -# Note: some sites will want to select more restrictive protections than the -# defaults. In particular, it may be desirable to switch many of the 644 -# entries to 640 or 600. For example, some sites will consider the -# contents of maillog, messages, and lpd-errs to be confidential. In the -# future, these defaults may change to more conservative ones. -# # logfilename [owner:group] mode count size when flags [/pid_file] [sig_num] -/var/log/all.log 600 7 * @T00 J -/var/log/amd.log 644 7 100 * J -/var/log/auth.log 600 7 100 @0101T JC -/var/log/console.log 600 5 100 * J -/var/log/cron 600 3 100 * JC -/var/log/daily.log 640 7 * @T00 JN -/var/log/debug.log 600 7 100 * JC -/var/log/init.log 644 3 100 * J -/var/log/kerberos.log 600 7 100 * J -/var/log/lpd-errs 644 7 100 * JC -/var/log/maillog 640 7 * @T00 JC -/var/log/messages 644 5 100 @0101T JC -/var/log/monthly.log 640 12 * $M1D0 JN -/var/log/pflog 600 3 100 * JB /var/run/pflogd.pid -/var/log/ppp.log root:network 640 3 100 * JC -/var/log/devd.log 644 3 100 * JC -/var/log/security 600 10 100 * JC +/var/log/all.log 600 7 * @T00 +/var/log/amd.log 640 7 100 * +/var/log/auth.log 600 7 100 @0101T C +/var/log/console.log 600 5 100 * +/var/log/cron 600 3 100 * C +/var/log/daily.log 640 7 * @T00 N +/var/log/debug.log 600 7 100 * C +/var/log/init.log 640 3 100 * +/var/log/kerberos.log 600 7 100 * +/var/log/lpd-errs 640 7 100 * C +/var/log/maillog 640 7 * @T00 C +/var/log/messages 640 5 100 @0101T C +/var/log/monthly.log 640 12 * $M1D0 N +/var/log/pflog 600 3 100 * B /var/run/pflogd.pid +/var/log/ppp.log root:network 640 3 100 * C +/var/log/devd.log 640 3 100 * C +/var/log/security 600 10 100 * C /var/log/sendmail.st 640 10 * 168 BN -/var/log/utx.log 644 3 * @01T05 B -/var/log/weekly.log 640 5 * $W6D0 JN -/var/log/xferlog 600 7 100 * JC +/var/log/utx.log 640 3 * @01T05 B +/var/log/weekly.log 640 5 * $W6D0 N +/var/log/xferlog 600 7 100 * C /etc/newsyslog.conf.d/* /usr/local/etc/newsyslog.conf.d/* -- 2.11.0 From e33516d181d2bd40a621f08c280924b60bf11a00 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 10:57:01 +0200 Subject: [PATCH 065/255] rc.d/jail: Remove obnoxious warning about 'obsolete' jail_* variables They will not be removed from ElectroBSD until a replacement exists that is usable with shell scripts like ezjail without jumping through lots of hoops. Obtained from: ElectroBSD --- etc/rc.d/jail | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/rc.d/jail b/etc/rc.d/jail index 8cf830a139a3..f9edc1a1bd79 100755 --- a/etc/rc.d/jail +++ b/etc/rc.d/jail @@ -15,7 +15,6 @@ desc="Manage system jails" rcvar="jail_enable" start_cmd="jail_start" -start_postcmd="jail_warn" stop_cmd="jail_stop" config_cmd="jail_config" console_cmd="jail_console" -- 2.11.0 From 11d05f2edd7d9340ee6c71227e57a191e6211dbb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 16:04:23 +0200 Subject: [PATCH 066/255] Don't build rcs papers until they built reproducible. Obtained from: ElectroBSD --- share/doc/psd/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/share/doc/psd/Makefile b/share/doc/psd/Makefile index 6b6d9cd727b2..9e0eed2082f5 100644 --- a/share/doc/psd/Makefile +++ b/share/doc/psd/Makefile @@ -20,7 +20,6 @@ SUBDIR= title \ 05.sysman \ 06.Clang \ 12.make \ - 13.rcs \ 15.yacc \ 16.lex \ 17.m4 \ -- 2.11.0 From 6fecf319278bd3ac857e0e11611a8f363e052962 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 12:34:11 +0200 Subject: [PATCH 067/255] kern.opts.mk: Disable IPFILTER, SOURCELESS_HOST and SOURCELESS_UCODE by default src.conf(5) will be updated once I find the time to figure out how to use tools/build/options/makeman. Obtained from: ElectroBSD --- sys/conf/kern.opts.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sys/conf/kern.opts.mk b/sys/conf/kern.opts.mk index 343b4f820161..f5913a7b5c36 100644 --- a/sys/conf/kern.opts.mk +++ b/sys/conf/kern.opts.mk @@ -33,21 +33,21 @@ __DEFAULT_YES_OPTIONS = \ FORMAT_EXTENSIONS \ INET \ INET6 \ - IPFILTER \ ISCSI \ KERNEL_SYMBOLS \ NETGRAPH \ PF \ - SOURCELESS_HOST \ - SOURCELESS_UCODE \ USB_GADGET_EXAMPLES \ ZFS __DEFAULT_NO_OPTIONS = \ EISA \ EXTRA_TCP_STACKS \ + IPFILTER \ NAND \ - OFED + OFED \ + SOURCELESS_HOST \ + SOURCELESS_UCODE # Some options are totally broken on some architectures. We disable # them. If you need to enable them on an experimental basis, you -- 2.11.0 From 728b3a4d4d8a8375261efa1fa11bdf112f74ac45 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 15 May 2015 13:11:57 +0200 Subject: [PATCH 068/255] boot/newvers.sh branding: Change bootprog_name[] to ElectroBSD Obtained from: ElectroBSD --- sys/boot/common/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/common/newvers.sh b/sys/boot/common/newvers.sh index ee2ac992e1dd..cd7422fa5430 100755 --- a/sys/boot/common/newvers.sh +++ b/sys/boot/common/newvers.sh @@ -40,7 +40,7 @@ u=${USER-root} h=${HOSTNAME-`hostname`} t=`date` #r=`head -n 6 $1 | tail -n 1 | awk -F: ' { print $1 } '` r=`awk -F: ' /^[0-9]\.[0-9]+:/ { print $1; exit }' $1` -echo "char bootprog_name[] = \"FreeBSD/${3} ${2}\";" > $tempfile +echo "char bootprog_name[] = \"ElectroBSD/${3} ${2}\";" > $tempfile echo "char bootprog_rev[] = \"${r}\";" >> $tempfile echo "char bootprog_date[] = \"${t}\";" >> $tempfile echo "char bootprog_maker[] = \"${u}@${h}\";" >> $tempfile -- 2.11.0 From 59d206c14d7b4121ad463a2afd64080ba46d44c5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 15:59:58 +0200 Subject: [PATCH 069/255] release: Default to not distributing the ports tree Allow to overwrite this with WITH_PORTS Obtained from: ElectroBSD --- release/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/Makefile b/release/Makefile index 60ffead48a54..a1c89cee63e6 100644 --- a/release/Makefile +++ b/release/Makefile @@ -23,9 +23,9 @@ # DOCDIR: location of doc tree (default: /usr/doc) # XTRADIR: xtra-bits-dir argument for /mkisoimages.sh # NOPKG: if set, do not distribute third-party packages -# NOPORTS: if set, do not distribute ports tree # NOSRC: if set, do not distribute source tree # NODOC: if set, do not generate release documentation +# WITH_PORTS: if set, distribute ports tree provided it exists # WITH_DVD: if set, generate dvd1.iso # WITH_COMPRESSED_IMAGES: if set, compress installation images with xz(1) # (uncompressed images are not removed) @@ -77,7 +77,7 @@ VOLUME_LABEL= FreeBSD_Install .if !exists(${DOCDIR}) NODOC= true .endif -.if !exists(${PORTSDIR}) +.if !exists(${PORTSDIR}) || !defined(WITH_PORTS) NOPORTS= true .endif -- 2.11.0 From 2cebbba97f83311f4efba0598718ebc85512f0f7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:26:44 +0200 Subject: [PATCH 070/255] sys/boot/common/newvers.sh: Allow to overwrite the date to make boot loader binaries reproducible Obtained from: ElectroBSD --- sys/boot/common/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/common/newvers.sh b/sys/boot/common/newvers.sh index cd7422fa5430..8c5e9d6cf685 100755 --- a/sys/boot/common/newvers.sh +++ b/sys/boot/common/newvers.sh @@ -36,7 +36,7 @@ tempfile=$(mktemp tmp.XXXXXX) || exit trap "rm -f $tempfile" EXIT INT TERM LC_ALL=C; export LC_ALL -u=${USER-root} h=${HOSTNAME-`hostname`} t=`date` +u=${USER-root} h=${HOSTNAME-`hostname`} t=${DATE-`date`} #r=`head -n 6 $1 | tail -n 1 | awk -F: ' { print $1 } '` r=`awk -F: ' /^[0-9]\.[0-9]+:/ { print $1; exit }' $1` -- 2.11.0 From 412428e1e9616ac332cb6c891e73ad5edba91b9b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:36:48 +0200 Subject: [PATCH 071/255] sys/conf/newvers.sh: Allow to overwrite the build date embedded into the kernel This is a required step to get reproducible builds. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 95a55cb8608a..22299eccc76a 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -101,7 +101,12 @@ v=`cat version` u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} -if [ -n "$SOURCE_DATE_EPOCH" ]; then +if [ -n "${DATE}" ]; then + # SOURCE_DATE_EPOCH was added upstream in r291691 + # but the ElectroBSD build goo is still setting DATE + # which has a different format. + t=${DATE} +elif [ -n "$SOURCE_DATE_EPOCH" ]; then if ! t=`date -r $SOURCE_DATE_EPOCH 2>/dev/null`; then echo "Invalid SOURCE_DATE_EPOCH" >&2 exit 1 -- 2.11.0 From 564b0df3193c5f4b1b44ca37a15ebfd99d85f437 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:10:44 +0200 Subject: [PATCH 072/255] amd64/make-memstick.sh: Create more 'predictable' images According to the mkimg man page, the "-y option is used for testing purposes only and is not to be used in production", but it feels good to be a gangsta. Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 52e17cc97b5f..d9590d1d08cb 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -38,6 +38,6 @@ fi rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local -mkimg -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.11.0 From feb1a16e946cd5a1363aace00d84ad028582f28f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:17:14 +0200 Subject: [PATCH 073/255] release/Makefile: Don't create matroshka src tarballs that contain other tarballs ... if DESTDIR isn't set to a reasonable value. While at it, exclude *.orig files as well. Obtained from: ElectroBSD --- release/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index a1c89cee63e6..fbc252ea486f 100644 --- a/release/Makefile +++ b/release/Makefile @@ -148,7 +148,9 @@ src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist usr/src | \ + --exclude .git --exclude @ --exclude usr/src/release/dist \ + --exclude "usr/src/release/*.txz" \ + --exclude "usr/src/release/*.orig" usr/src | \ ${XZ_CMD} > ${.OBJDIR}/src.txz ports.txz: -- 2.11.0 From 02be34e0620ae710bdb88446444372601d5a0ea4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 18:45:24 +0200 Subject: [PATCH 074/255] release: Use a hack to recreate dist tarballs with reproducible timestamps Obtained from: ElectroBSD --- release/Makefile | 3 ++ release/scripts/tar-time-reset.sh | 64 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100755 release/scripts/tar-time-reset.sh diff --git a/release/Makefile b/release/Makefile index fbc252ea486f..627b6ea6bd7d 100644 --- a/release/Makefile +++ b/release/Makefile @@ -266,6 +266,9 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} + for tarball in *.txz; do \ + sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ + done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh new file mode 100755 index 000000000000..8186fd8d3e26 --- /dev/null +++ b/release/scripts/tar-time-reset.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# This script resets the timestamps in a given tarfile to hopefully make +# it reproducible. This is a rather wasteful approach, but works for now. +# +# A better solution would be to patch bsdtar to optionally use a fixed +# time (without having to clown around with mtree specs). +# +########################################################################## + +main() { + local tarfile="${1}" \ + tempdir mtree_spec + + if [ $# -ne 1 ]; then + echo "$0 /path/to/tarfile" + exit 1 + fi + + if [ -z "${tarfile}" ]; then + echo "No tar file given" + return 1 + fi + tarfile="$(realpath "$tarfile")" + + tempdir=$(mktemp -d) || return 1 + mtree_spec=$(mktemp) || return 1 + + echo "Extracting tarfile ${tarfile}" + (cd "${tempdir}" && tar xvf "${tarfile}") || return 1 + + echo "Ditching original tarfile ${tarfile}" + rm "${tarfile}" + + echo "Creating mtree spec in ${mtree_spec}" + (cd "${tempdir}" && mtree -L -c -k time) | \ + sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${mtree_spec}" + + echo "Creating tarfile ${tarfile}" + (cd "${tempdir}" && tar acLvf "${tarfile}" @"${mtree_spec}") || return 1 + + echo "Ditching ${tempdir}" + rm -r "${tempdir}" || return 1 + echo "Ditching ${mtree_spec}" + rm "${mtree_spec}" || return 1 +} + +main "${@}" -- 2.11.0 From ee684de8c98e64fc50883e5ccd6b9b6f520d38ca Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 May 2015 16:06:01 +0200 Subject: [PATCH 075/255] release/amd64/make-memstick.sh: Use reproducible timestamps for the makefs image Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index d9590d1d08cb..a5d88e166e23 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -30,13 +30,15 @@ fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -makefs -B little -o label=FreeBSD_Install ${2}.part ${1} +mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 fi rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local +rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.11.0 From a3c826850374603fe05c772a0725da3d8c7b7cbf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 14:25:17 +0200 Subject: [PATCH 076/255] Remove build timestamps from ntp* binaries again This was already done by r195626 a couple of years ago apparently the change got lost while importing an update from the vendor. Obtained from: ElectroBSD --- contrib/ntp/scripts/build/mkver.in | 2 -- 1 file changed, 2 deletions(-) mode change 100644 => 100755 contrib/ntp/scripts/build/mkver.in diff --git a/contrib/ntp/scripts/build/mkver.in b/contrib/ntp/scripts/build/mkver.in old mode 100644 new mode 100755 index 3aef1c816a7f..badd2cda8c1f --- a/contrib/ntp/scripts/build/mkver.in +++ b/contrib/ntp/scripts/build/mkver.in @@ -15,8 +15,6 @@ case "@VER_SUFFIX@" in *) ConfStr="${ConfStr}-@VER_SUFFIX@" ;; esac -ConfStr="$ConfStr `LC_TIME=C TZ=UTC date`" - if [ ! -f .version ]; then echo 0 > .version fi -- 2.11.0 From c51625aed1762fe91820382f505e68c2dd7f4455 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 12:43:25 +0200 Subject: [PATCH 077/255] usr.sbin/ntp: Allow to set MKREPRO_DATE and MKREPRO_TIME to get reproducible builds Obtained from: ElectroBSD --- usr.sbin/ntp/libntp/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/usr.sbin/ntp/libntp/Makefile b/usr.sbin/ntp/libntp/Makefile index 74babee5d841..f7c82f345af4 100644 --- a/usr.sbin/ntp/libntp/Makefile +++ b/usr.sbin/ntp/libntp/Makefile @@ -85,6 +85,13 @@ CFLAGS+= -I${.CURDIR}/../../../contrib/ntp/include \ CFLAGS+= -DHAVE_BSD_NICE -DHAVE_STDINT_H +.if defined(MKREPRO_DATE) +CFLAGS+= -DMKREPRO_DATE="\"${MKREPRO_DATE}\"" +.endif +.if defined(MKREPRO_TIME) +CFLAGS+= -DMKREPRO_TIME="\"${MKREPRO_TIME}\"" +.endif + CLEANFILES+= .version version.c version.c: -- 2.11.0 From bbf6adb8e4eb89d00fbee7b5c89887b85f1dd385 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 May 2015 11:33:50 +0200 Subject: [PATCH 078/255] release: Allow to build the mtree spec for mergemaster reproducible This relies on NetBSD mtree which has been the default for a while now. Obtained from: ElectroBSD --- release/scripts/mm-mtree.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release/scripts/mm-mtree.sh b/release/scripts/mm-mtree.sh index 620ab6dd962c..279220b78319 100755 --- a/release/scripts/mm-mtree.sh +++ b/release/scripts/mm-mtree.sh @@ -146,8 +146,12 @@ find ${TEMPROOT} -type f -size 0 -delete 2>/dev/null find -d ${TEMPROOT} -type d -empty -delete 2>/dev/null # Build the mtree database in a temporary location. +# The second mtree call is used to get a reproducible result +# without embedded hostname, user name and creation timestamp. +# XXX: Might be obsolete after r301584. MTREENEW=`mktemp -t mergemaster.mtree` -mtree -nci -p ${TEMPROOT} -k size,md5digest > ${MTREENEW} 2>/dev/null +mtree -mci -p ${TEMPROOT} -k size,md5digest 2>/dev/null | \ + mtree -C -k all > ${MTREENEW} if [ -s "${MTREENEW}" ]; then echo "*** Saving mtree database for future upgrades" -- 2.11.0 From 8c4cf47d6e2e4bf16cc31622811d06ce392eee02 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 12:07:25 +0200 Subject: [PATCH 079/255] Make reproducing builds more convenient ... by setting the various variables based on the environment variable REPRO_SEED. Obtained from: ElectroBSD --- Makefile.inc1 | 14 +++++++--- release/Makefile | 2 ++ share/mk/src.reproducible-build.mk | 54 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 share/mk/src.reproducible-build.mk diff --git a/Makefile.inc1 b/Makefile.inc1 index dc1d0a440be9..a06f629ec333 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -48,6 +48,8 @@ .error "Both TARGET and TARGET_ARCH must be defined." .endif +.include "share/mk/src.reproducible-build.mk" + SRCDIR?= ${.CURDIR} LOCALBASE?= /usr/local @@ -790,17 +792,23 @@ WMAKE_TGTS+= build${libcompat} buildworld: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue .PHONY .ORDER: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue -buildworld_prologue: .PHONY +buildworld_prologue: .PHONY reproducible_build_hint @echo "--------------------------------------------------------------" @echo ">>> World build started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" -buildworld_epilogue: .PHONY +buildworld_epilogue: .PHONY reproducible_build_hint @echo @echo "--------------------------------------------------------------" @echo ">>> World build completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" +reproducible_build_hint: + @echo "--------------------------------------------------------------" + @echo ">>> To reproduce this build:" + @echo ">>> export REPRO_SEED=$${REPRO_SEED}" + @echo "--------------------------------------------------------------" + # # We need to have this as a target because the indirection between Makefile # and Makefile.inc1 causes the correct PATH to be used, rather than a @@ -1170,7 +1178,7 @@ ${WMAKE_TGTS:N_worldtmp:Nbuild${libcompat}} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE # # Builds all kernels defined by BUILDKERNELS. # -buildkernel: .MAKE .PHONY +buildkernel: .MAKE .PHONY reproducible_build_hint .if empty(BUILDKERNELS:Ndummy) @echo "ERROR: Missing kernel configuration file(s) (${KERNCONF})."; \ false diff --git a/release/Makefile b/release/Makefile index 627b6ea6bd7d..7fa748246b37 100644 --- a/release/Makefile +++ b/release/Makefile @@ -36,6 +36,8 @@ # TARGET/TARGET_ARCH: architecture of built release # +.include "../share/mk/src.reproducible-build.mk" + WORLDDIR?= ${.CURDIR}/.. PORTSDIR?= /usr/ports DOCDIR?= /usr/doc diff --git a/share/mk/src.reproducible-build.mk b/share/mk/src.reproducible-build.mk new file mode 100644 index 000000000000..dc4f4df93507 --- /dev/null +++ b/share/mk/src.reproducible-build.mk @@ -0,0 +1,54 @@ +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# Make the build reproducible by exporting a bunch of variables, +# potentionally using an already-set REPRO_SEED as input. +# +# The variable names are somewhat stupid, mostly because we +# are using existing ones. +# +########################################################################## + +TZ= "UTC" +.export TZ + +.if ! defined(REPRO_SEED) +# XXX: Currently we don't add the kernel version number to the repro +# seed because we expect a clean object tree in which case it +# will reproducible be 0. If the object tree of a the +# build-to-reproduce was actually unclean, KERNEL_VERSION_NUMBER +# has to be set to a matching value. +REPRO_SEED!= echo $$(id -un):$$(hostname):$$(date +%s) +.export REPRO_SEED +.else +.if ! defined(KERNEL_VERSION_NUMBER) +KERNEL_VERSION_NUMBER=0 +.export KERNEL_VERSION_NUMBER +.endif +.endif + +USER!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 1 +HOSTNAME!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 2 +EPOCH_DATE!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 3 + +DATE!= date -r ${EPOCH_DATE} +# These two probably are no longer necessary after r285701 +MKREPRO_DATE!= date -r ${EPOCH_DATE} +"%b %d %Y" +MKREPRO_TIME!= date -r ${EPOCH_DATE} +%H:%M:%S + +.for v in REPRO_SEED USER HOSTNAME EPOCH_DATE DATE MKREPRO_DATE MKREPRO_TIME +.export $v +.endfor -- 2.11.0 From 8e10c447a2e28e5d02cce26261e9e08f67fddd8e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 23 May 2015 20:13:18 +0200 Subject: [PATCH 080/255] contrib/groff/mdate.sh: Use ${EPOCH_DATE} when set While this is silly, using the time the man pages have been last checked out from the VCS is silly as well, so this commit doesn't increase the total amount of sillyness in the system. Also we are talking about GNU roff here, so hopefully this stuff will be garbage-collected in the near future. Obtained from: ElectroBSD --- contrib/groff/mdate.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/groff/mdate.sh b/contrib/groff/mdate.sh index 4a26e6e80bad..615abc784123 100755 --- a/contrib/groff/mdate.sh +++ b/contrib/groff/mdate.sh @@ -1,12 +1,17 @@ #! /bin/sh # Print the modification date of $1 `nicely'. +# If ${EPOCH_DATE} is set, it will be used instead. # Don't want foreign dates. LANGUAGE= LC_ALL=C; export LC_ALL +if [ -n "${EPOCH_DATE}" ]; then + date -r "${EPOCH_DATE}" +"%d %B %Y" + exit 0 +fi (date; if ls -L /dev/null 1>/dev/null 2>&1; then ls -L -l $1; else ls -l $1; fi -- 2.11.0 From 2c49b30a595fee0ce0c914a15259a17a135a8fcb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 May 2015 18:01:31 +0200 Subject: [PATCH 081/255] Fake modification time smarter for everything but EXTRA_PACKAGES (XXX) While at it, remove duplicated slashes in the METALOG as they result in missing files. XXX: Should use g flag. Mark two suspicious mtree spec modifications as such. XXX: investigate. Obtained from: ElectroBSD --- Makefile.inc1 | 18 ++++++++++++++++++ release/Makefile | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index a06f629ec333..4b6c8d77834f 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1022,6 +1022,14 @@ distributeworld installworld stageworld: _installcheck_world .PHONY find ${DESTDIR}/${DISTDIR}/${dist} -mindepth 1 -type d -empty -delete .endfor .if defined(NO_ROOT) + @# Post process METALOG: add fake timestamps and, if necessary, + @# remove duplicated slashes which can occur if DISTDIR is undefined. + @# Keeping them results in missing files in the distribution tarballs. +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9\.]+@@' \ + -e 's@(type=)@uid=0 gid=0 time=${EPOCH_DATE}.0 \1@' \ + -e 's@//@/@' -i '.bak' ${METALOG} +.endif .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @@ -1303,6 +1311,11 @@ distributekernel distributekernel.debug: .PHONY packagekernel: .PHONY .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) +.if defined(EPOCH_DATE) +# XXX: Is this really necessary given that we already modify the METALOG itself? + sed -E -e 's@time=[0-9\.]+@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.meta | \ @@ -1314,6 +1327,11 @@ packagekernel: .PHONY ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} +# XXX: See XXX above +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9\.]+@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ diff --git a/release/Makefile b/release/Makefile index 7fa748246b37..e227d19aa559 100644 --- a/release/Makefile +++ b/release/Makefile @@ -56,6 +56,10 @@ TARGET_ARCH= ${TARGET} IMAKE= ${MAKE} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET} DISTDIR= dist +# Enable mtree spec usage so we can fake the modification time. +NO_ROOT=1 +.export NO_ROOT + # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -268,7 +272,7 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in *.txz; do \ + for tarball in ${EXTRA_PACKAGES}; do \ sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST -- 2.11.0 From 0743905506151e59dccaa35710c1b2366977a5c8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 10:27:48 +0200 Subject: [PATCH 082/255] Add image-checksum.sh ... which calculates a checksum of the reproducible parts of an memstick image. Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 156 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100755 release/scripts/image-checksum.sh diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh new file mode 100755 index 000000000000..6f79a81ae37a --- /dev/null +++ b/release/scripts/image-checksum.sh @@ -0,0 +1,156 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# image-checksum.sh /path/to/memstick.img +# +# Unfortunately the memstick target currently does not create reproducible +# ElectroBSD images due to unreproducible differences in the file system +# layer of the data partition. +# +# To be able to (sort of) compare memstick images anyway, this script +# produces a "partial image checksum" that is based on the partition layout, +# the checksum of the boot code partition and an mtree spec of the data +# partition which includes checksums, sizes and timestamps for all the +# files. +# +# A memstick image whose "partial checksum" matches the one of another +# image can be totally considered to be nearly as trustworthy. Obviously +# that's a somewhat worthless property, it is thus recommended that you +# rebuild the potentionally malicious image using a trusted operating +# system first. After you've done this, potentionally malicious differences +# in the unchecked parts should be gone. +# +# Just kidding, image-checksum.sh is only intended to regression-test +# the ElectroBSD build system. +# +# Also note that this script relies on non-standardized output of other +# tools which might occasionally change. To be able to reproduce partial +# image checksums you thus need a userland that is close enough to the +# one that was used to create the original version. +# +########################################################################## + +UFS_PARTITION=p2 +EXPECTED_PARTITIONS=2 +MOUNTPOINT=/mnt +VERBOSE=0 +MTREE_KEYWORDS=size,time,uid,gid,sha256 + +verbose_log() { + local message="$*" + if [ "${VERBOSE}" = 0 ]; then + return + fi + echo "${message}" +} + +create_mtree_spec_file() { + local md_unit spec_file + + md_unit="${1}" + spec_file="${2}" + + verbose_log "Mounting /dev/md${md_unit}${UFS_PARTITION} at ${MOUNTPOINT}" + mount -o ro "/dev/md${md_unit}${UFS_PARTITION}" "${MOUNTPOINT}" || return 1 + + verbose_log "Running mtree, saving spec in ${spec_file}" + mtree -c -k "${MTREE_KEYWORDS}" -p "${MOUNTPOINT}" | mtree -C -k all > "${spec_file}" || return 1 + + verbose_log "Unmounting ${MOUNTPOINT} ..." + umount "${MOUNTPOINT}" || return 1 +} + +partition_count_acceptable() { + local md_unit="${1}" + + # Verify that there are exactly two partitions present + partitions=$(gpart show -r -p "md${md_unit}" | grep -c "md${md_unit}"p) + if [ "${partitions}" != "${EXPECTED_PARTITIONS}" ]; then + echo "Invalid number of partitions: ${partitions}" + return 1; + fi +} + +main() { + local image_file \ + args md_unit spec_file gpart_file + + args=$(getopt v $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + image_file=${1} + if [ -z "${image_file}" ]; then + echo "No image file provided" + return 1 + fi + spec_file="${image_file}.mtree" + if [ -f "${spec_file}" ]; then + echo "Spec file ${spec_file} already exists" + return 1 + fi + gpart_file="${image_file}.gpart" + if [ -f "${spec_file}" ]; then + echo "gpart file ${gpart_file} already exists" + return 1 + fi + + md_unit=$(mdconfig -o readonly -n -f "${image_file}") + if [ $? != 0 ]; then + return 1 + fi + + partition_count_acceptable "${md_unit}" || return 1 + + if [ ! -f "${spec_file}" ]; then + create_mtree_spec_file "${md_unit}" "${spec_file}" || return 1 + fi + if [ ! -f "${gpart_file}" ]; then + gpart list "md${md_unit}" | sed -E -e "s@(: md)${md_unit}@\1X@" > "${gpart_file}" + fi + + gpart_checksum=$(sha256 -q "${gpart_file}") + verbose_log "gpart checksum: ${gpart_checksum}" + + mdconfig -d -u "${md_unit}" || return 1 + + bootcode_checksum=$(dd if=/dev/md${md_unit}p1 2>/dev/null | sha256) + verbose_log "Boot code checksum: ${bootcode_checksum}" + mtree_checksum=$(sha256 -q "${spec_file}") + verbose_log "mtree checksum: ${mtree_checksum}" + + weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) + echo "Partial image checksum: ${weak_image_checksum}" + +} + +main "${@}" -- 2.11.0 From 156add810a5350706a2dbbad2950343fbe38775f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 19:54:10 +0200 Subject: [PATCH 083/255] image-checksum: Add -r flag to reuse cache files Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 6f79a81ae37a..1940921b44a1 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -50,6 +50,7 @@ EXPECTED_PARTITIONS=2 MOUNTPOINT=/mnt VERBOSE=0 MTREE_KEYWORDS=size,time,uid,gid,sha256 +REUSE_EXISTING_CACHE_FILES=false verbose_log() { local message="$*" @@ -90,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt v $*) + args=$(getopt rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -98,6 +99,10 @@ main() { set -- ${args} while true; do case "$1" in + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; -v) VERBOSE=1 shift @@ -116,12 +121,12 @@ main() { spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi gpart_file="${image_file}.gpart" if [ -f "${spec_file}" ]; then echo "gpart file ${gpart_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi md_unit=$(mdconfig -o readonly -n -f "${image_file}") -- 2.11.0 From 485ea68c6c1ba3269a66950d61f4fe7095724da0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 20:20:01 +0200 Subject: [PATCH 084/255] image-checksum.sh: Allow to overwrite the mtree flags Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 1940921b44a1..cc31e9055fe3 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -91,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt rv $*) + args=$(getopt m:rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -99,6 +99,11 @@ main() { set -- ${args} while true; do case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; -r) REUSE_EXISTING_CACHE_FILES=true shift -- 2.11.0 From 201a0e274a89ff42efa9e4127f2e7c22ca432919 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 31 Aug 2015 10:51:16 +0200 Subject: [PATCH 085/255] release/scripts/image-checksum.sh: Allow to checksum multiple images at once Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 79 ++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index cc31e9055fe3..30ac62a6c168 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -87,42 +87,11 @@ partition_count_acceptable() { fi } -main() { +generate_partial_image_checksum() { local image_file \ - args md_unit spec_file gpart_file - - args=$(getopt m:rv $*) - if [ $? -ne 0 ]; then - echo 'You are doing it wrong: Invalid flag specified' - exit 2 - fi - set -- ${args} - while true; do - case "$1" in - -m) - shift - MTREE_KEYWORDS="${1}" - shift - ;; - -r) - REUSE_EXISTING_CACHE_FILES=true - shift - ;; - -v) - VERBOSE=1 - shift - ;; - --) - shift; break - ;; - esac - done + md_unit spec_file gpart_file - image_file=${1} - if [ -z "${image_file}" ]; then - echo "No image file provided" - return 1 - fi + image_file="${1}" spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" @@ -159,8 +128,48 @@ main() { verbose_log "mtree checksum: ${mtree_checksum}" weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) - echo "Partial image checksum: ${weak_image_checksum}" + echo "Partial image checksum for ${image_file}: ${weak_image_checksum}" +} +main() { + local image_file \ + args + + args=$(getopt m:rv $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + if [ -z "${1}" ]; then + echo "No image file provided" + return 1 + fi + + for image_file in "${@}"; do + generate_partial_image_checksum "${image_file}" || return 1 + done } main "${@}" -- 2.11.0 From bd388e3060dcefa8b4bf153418a6a175ba126452 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:38:48 +0200 Subject: [PATCH 086/255] sys: Do not embed the compiler version in the kernel binary ... as it makes reproducing the binary with a different compiler more complicated. In case of ElectroBSD the compiler used can be usually deduced from the uname output anyway as the upstream revision is part of the fake hostname when using reproduce.sh. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 2 -- sys/kern/init_main.c | 1 - sys/kern/kern_mib.c | 3 --- sys/sys/systm.h | 1 - 4 files changed, 7 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 22299eccc76a..a11a6bfe3573 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -115,7 +115,6 @@ else t=`date` fi i=`${MAKE:-make} -V KERN_IDENT` -compiler_v=$($(${MAKE:-make} -V CC) -v 2>&1 | grep -w 'version') for dir in /usr/bin /usr/local/bin; do if [ ! -z "${svnversion}" ] ; then @@ -240,7 +239,6 @@ $COPYRIGHT char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR; char version[sizeof(VERSTR) > 256 ? sizeof(VERSTR) : 256] = VERSTR; -char compiler_version[] = "${compiler_v}"; char ostype[] = "${TYPE}"; char osrelease[sizeof(RELSTR) > 32 ? sizeof(RELSTR) : 32] = RELSTR; int osreldate = ${RELDATE}; diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 85382cdd8c51..c416c9acf7c6 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -341,7 +341,6 @@ print_version(void *data __unused) while (len > 0 && version[len - 1] == '\n') len--; printf("%.*s %s\n", len, version, machine); - printf("%s\n", compiler_version); } SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t, diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c index 875d2b97c998..ab013666306b 100644 --- a/sys/kern/kern_mib.c +++ b/sys/kern/kern_mib.c @@ -97,9 +97,6 @@ SYSCTL_INT(_kern, KERN_OSREV, osrevision, CTLFLAG_RD|CTLFLAG_CAPRD, SYSCTL_STRING(_kern, KERN_VERSION, version, CTLFLAG_RD|CTLFLAG_MPSAFE, version, 0, "Kernel version"); -SYSCTL_STRING(_kern, OID_AUTO, compiler_version, CTLFLAG_RD|CTLFLAG_MPSAFE, - compiler_version, 0, "Version of compiler used to compile kernel"); - SYSCTL_STRING(_kern, KERN_OSTYPE, ostype, CTLFLAG_RD|CTLFLAG_MPSAFE| CTLFLAG_CAPRD, ostype, 0, "Operating system type"); diff --git a/sys/sys/systm.h b/sys/sys/systm.h index caffd7baafdf..90d633684497 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -50,7 +50,6 @@ extern int suspend_blocked; /* block suspend due to pending shutdown */ extern int rebooting; /* kern_reboot() has been called. */ extern const char *panicstr; /* panic message */ extern char version[]; /* system version */ -extern char compiler_version[]; /* compiler version */ extern char copyright[]; /* system copyright */ extern int kstack_pages; /* number of kernel stack pages */ -- 2.11.0 From 8c53b5edd0e9d738894f64fa99c839e5ecd137ed Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:42:43 +0200 Subject: [PATCH 087/255] sys/conf/newvers.sh: Ditch support for p4 (non-free) and hg (not relevant for ElectroBSD) Obtained from: ElectroBSD --- sys/conf/newvers.sh | 46 ++-------------------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index a11a6bfe3573..bc688dd63799 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -141,11 +141,6 @@ if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then fi fi -for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/p4" ] && [ -z ${p4_cmd} ] ; then - p4_cmd=${dir}/p4 - fi -done if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -155,15 +150,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -d "${SYSDIR}/../.hg" ] ; then - for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/hg" ] ; then - hg_cmd="${dir}/hg -R ${SYSDIR}/.." - break - fi - done -fi - if [ -n "$svnversion" ] ; then svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` case "$svn" in @@ -203,38 +189,10 @@ if [ -n "$git_cmd" ] ; then # fi fi -if [ -n "$p4_cmd" ] ; then - p4version=`cd ${SYSDIR} && $p4_cmd changes -m1 "./...#have" 2>&1 | \ - awk '{ print $2 }'` - case "$p4version" in - [0-9]*) - p4version=" ${p4version}" - p4opened=`cd ${SYSDIR} && $p4_cmd opened ./... 2>&1` - case "$p4opened" in - File*) ;; - //*) p4version="${p4version}+edit" ;; - esac - ;; - *) unset p4version ;; - esac -fi - -if [ -n "$hg_cmd" ] ; then - hg=`$hg_cmd id 2>/dev/null` - svn=`$hg_cmd svn info 2>/dev/null | \ - awk -F': ' '/Revision/ { print $2 }'` - if [ -n "$svn" ] ; then - svn=" r${svn}" - fi - if [ -n "$hg" ] ; then - hg=" ${hg}" - fi -fi - cat << EOF > vers.c $COPYRIGHT -#define SCCSSTR "@(#)${VERSION} #${v}${svn}${git}${hg}${p4version}: ${t}" -#define VERSTR "${VERSION} #${v}${svn}${git}${hg}${p4version}: ${t}\\n ${u}@${h}:${d}\\n" +#define SCCSSTR "@(#)${VERSION} #${v}${svn}${git}: ${t}" +#define VERSTR "${VERSION} #${v}${svn}${git}: ${t}\\n ${u}@${h}:${d}\\n" #define RELSTR "${RELEASE}" char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR; -- 2.11.0 From 79bd5d985fde0f7208aba90b674df4ab00438056 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 3 Jun 2015 17:16:10 +0200 Subject: [PATCH 088/255] mandocdb: Normalize inodevs to get reproducible results This is a proof-of-concept that scales poorly due to linear searches. The proper fix is probably to use hash-based lookups and only do it when the user wants reproducible results. I didn't find any automated tests for this so the only testing done is "does 'man' appear to be usable". It indeed appears to be ... Obtained from: ElectroBSD --- contrib/mdocml/mandocdb.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/contrib/mdocml/mandocdb.c b/contrib/mdocml/mandocdb.c index 3b9bda0d612b..5bdbb563c6c5 100644 --- a/contrib/mdocml/mandocdb.c +++ b/contrib/mdocml/mandocdb.c @@ -946,6 +946,34 @@ filescan(const char *file) mlink_add(mlink, &st); } +/* + * Messes up inodevs in a reproducible way as long as + * the call order does not change. The implementation + * is silly and only used as proof of concept. + */ +#define HOPEFULLY_ENOUGH_FOR_EVERYBODY 5000 +static void +normalize_inodev(struct inodev *inodev) { + static size_t table[HOPEFULLY_ENOUGH_FOR_EVERYBODY]; + size_t key; + int i; + + key = inodev->st_ino + inodev->st_dev; + + for (i = 0; i < sizeof(table)/sizeof(table[0]); i++) { + if (table[i] == 0) { + /* New value, add to table*/ + table[i] = key; + } + if (table[i] == key) { + /* Use index as new value */ + inodev->st_ino = i; + inodev->st_dev = i; + return; + } + } +} + static void mlink_add(struct mlink *mlink, const struct stat *st) { @@ -976,6 +1004,7 @@ mlink_add(struct mlink *mlink, const struct stat *st) memset(&inodev, 0, sizeof(inodev)); /* Clear padding. */ inodev.st_ino = st->st_ino; inodev.st_dev = st->st_dev; + normalize_inodev(&inodev); slot = ohash_lookup_memory(&mpages, (char *)&inodev, sizeof(struct inodev), inodev.st_ino); mpage = ohash_find(&mpages, slot); -- 2.11.0 From 46a4ac3f47454cf190fcbb14efbc5b57853f9187 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 1 Jun 2015 13:56:05 +0200 Subject: [PATCH 089/255] release/Makefile: Build the src.txz only once and also fake source ownership If taring the sources fails, keep the mtree spec. Keep release/scripts/tar-time-reset.sh for now but update a comment to make it obvious that the script isn't used anymore Obtained from: ElectroBSD --- release/Makefile | 28 ++++++++++++++++++++-------- release/scripts/tar-time-reset.sh | 8 +++----- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/release/Makefile b/release/Makefile index e227d19aa559..335486c2bacf 100644 --- a/release/Makefile +++ b/release/Makefile @@ -153,11 +153,26 @@ kernel.txz: src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src - cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist \ - --exclude "usr/src/release/*.txz" \ - --exclude "usr/src/release/*.orig" usr/src | \ - ${XZ_CMD} > ${.OBJDIR}/src.txz +# Create an mtree spec with faked timestamps so we get a reproducible +# tar file. We do not use tar for this because its mtree generator +# appears to be buggy and exits with an memory allocation failure. +# +# It's important that the excluded paths start with "./", otherwise +# file locations are not recorded correctly, and, for example, +# usr/src/usr.sbin appears as usr/src/release/usr.sbin in the +# tar file. Only the shadow knows if that's a bug or a feature. + echo "./usr/src/.git" >${.OBJDIR}/mtree-exclude + echo "./usr/src/release/dist" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/src.mtree" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/mtree-exclude" >>${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && mtree -c -L -k time -X ${.OBJDIR}/mtree-exclude | \ + mtree -C | \ + sed -E -e 's@time=[0-9]+\.[0-9]+@time=${EPOCH_DATE}.0 uid=0 gid=0@' \ + > ${.OBJDIR}/src.mtree + rm ${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && tar cLvf - @${.OBJDIR}/src.mtree \ + | ${XZ_CMD} > ${.OBJDIR}/src.txz && \ + rm ${.OBJDIR}/src.mtree ports.txz: mkdir -p ${DISTDIR}/usr @@ -272,9 +287,6 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in ${EXTRA_PACKAGES}; do \ - sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ - done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh index 8186fd8d3e26..f1b84340286d 100755 --- a/release/scripts/tar-time-reset.sh +++ b/release/scripts/tar-time-reset.sh @@ -17,11 +17,9 @@ ########################################################################## # # This script resets the timestamps in a given tarfile to hopefully make -# it reproducible. This is a rather wasteful approach, but works for now. -# -# A better solution would be to patch bsdtar to optionally use a fixed -# time (without having to clown around with mtree specs). -# +# it reproducible. As this is a rather wasteful approach the script is +# no longer used. It hasn't been removed yet as it may be useful for +# testing purposes. ########################################################################## main() { -- 2.11.0 From 3ce9ce3fb6219b274c3c88f7ab738e51b6368b21 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Jun 2015 12:18:16 +0200 Subject: [PATCH 090/255] release/Makefile: Reorder dependencies to reduce the chances that base or kernel parts end up in the src.txz. This is merely a workaround Obtained from: ElectroBSD --- release/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index 335486c2bacf..83d46a499b55 100644 --- a/release/Makefile +++ b/release/Makefile @@ -286,7 +286,7 @@ mini-memstick: mini-memstick.img mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} -packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} +packagesystem: ${EXTRA_PACKAGES} base.txz kernel.txz sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} -- 2.11.0 From 41623e2db1314b589492319c387cf268771f2c6f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 17:30:25 +0200 Subject: [PATCH 091/255] Add strip-freebsd.sh ... which suggests a bunch of stuff to delete from a vanilla FreeBSD checkout. In a previous life it was called free-freebsd.sh which obviously sounds more awesome, but nowadays most of the suggested stuff for removal is actually free software that just isn't relevant for ElectroBSD. Removing code we don't need means we don't have to care about its security and license issues. There's lots of code to remove left! Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 171 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100755 release/scripts/strip-freebsd.sh diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh new file mode 100755 index 000000000000..eaaf3789a0ab --- /dev/null +++ b/release/scripts/strip-freebsd.sh @@ -0,0 +1,171 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# This script pretends to free a FreeBSD checkout from +# known-unfree files and other stuff that is not required +# by ElectroBSD. +# +# While it is pretty much guaranteed to ditch a bunch of files +# it doesn't work very thorougly and the resulting checkout is +# likely to still contain lots of non-free parts that haven't +# been discovered yet. + +get_snd_csa_files() { + find sys/dev/sound/pci -name "csa*" +} + +get_snd_ds1_files() { + find sys/dev/sound/pci -name "ds1*" +} + +get_snd_maestro3_files() { + # This file contains the actual blobs + echo sys/dev/sound/pci/allegro_code.h + + # This file contains the (free) code that relies on the blobs. + echo sys/dev/sound/pci/maestro3.c +} + +# We can't simply remove the whole directory +# as a (free) header is required by bge. +get_bce_files() { + echo "sys/dev/bce/if_bcefw.h" \ + "sys/dev/bce/if_bce.c" +} + +get_usb_firmware_files() { + find sys/dev/usb/ -name "*fw*" +} + +# XXX: Misleading name, some of the files are merely tainted +# by non-free dependencies +get_unfree_files() { + find sys/ -name "*.uu" + get_snd_csa_files + get_snd_ds1_files + get_snd_maestro3_files + get_bce_files + get_usb_firmware_files +} + +get_files_to_ditch() { + get_unfree_files +} + +# These architectures are mainly unsupported by ElectroBSD +# due to lack of hardware for testing purposes. +# +# The source directories are mainly removed to shrink the +# source tarball and to reduce the number of files that +# should be audited for license and security issues. +get_unsupported_architectures() { + echo "arm arm64 mips pc98 powerpc sparc64" +} + +# These depend on or contain proprietary firmware that is included in sys/contrib/dev +get_tainted_sys_contrib_devs() { + echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu run uath urtwn wpi" +} + +# These require proprietary firmware that is included in sys/dev +# and may cause build failures without it. +get_tainted_sys_devs() { + # bce has already been taken care of by get_bce_files() above + echo "bxe ctau cx cxgb cxgbe ispfw qlxgbe" \ + "spibus it tw" +} + +get_unused_contrib_dirs() { + # XXX: gcc can't be deleted because parts of it are apparently + # required to build libc. This should be investigated more thoroughly, + # hopefully it can be fixed. + echo "apr apr-util ipfilter ofed sendmail serf subversion tcsh" +} + +get_directories_to_ditch() { + local arch \ + dir arch_dir sys_contrib contrib_dir + + for dir in sys sys/boot; do + for arch in $(get_unsupported_architectures); do + potential_directory="${dir}/${arch}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done + done + + for sys_contrib in ipfilter octeon-sdk; do + echo "sys/contrib/${sys_contrib}" + done + + for sys_contrib in $(get_tainted_sys_contrib_devs); do + echo "sys/contrib/dev/${sys_contrib}" + done + + for sys_dev in $(get_tainted_sys_devs); do + echo "sys/dev/${sys_dev}" + done + + for contrib_dir in $(get_unused_contrib_dirs); do + potential_directory="contrib/${contrib_dir}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done +} + +purify_cwd() { + # There are no spaces in paths or file names. + files_to_ditch="$(get_files_to_ditch)" + for f in $files_to_ditch; do + [ -f "${f}" ] && echo "rm ${f}" + done + + dirs_to_ditch="$(get_directories_to_ditch)" + for d in $dirs_to_ditch; do + [ -d "${d}" ] && echo "rm -r ${d}" + done +} + +main() { + local src_dir \ + files_to_ditch dirs_to_ditch + + src_dir="${1}" + if [ -z "${src_dir}" ]; then + echo "No source directory given" + return 1 + fi + if [ ! -d "${src_dir}" ]; then + echo "No such directory: ${src_dir}" + return 1 + fi + # Make it less likely to operate on a directory + # that isn't actually a FreeBSD checkout + if [ ! -f "${src_dir}/COPYRIGHT" ]; then + echo "${src_dir} contains no COPYRIGHT file" + return 1 + fi + + cd "${src_dir}" || return 1 + + purify_cwd +} + +main "${@}" -- 2.11.0 From b60ce8adcf9f13c33f22ed1be31bed8122b491c7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 1 Jan 2016 11:22:49 +0100 Subject: [PATCH 092/255] release/scripts/strip-freebsd.sh: Ditch rtwn which got added in r293009 Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index eaaf3789a0ab..56be187cca4c 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -79,7 +79,7 @@ get_unsupported_architectures() { # These depend on or contain proprietary firmware that is included in sys/contrib/dev get_tainted_sys_contrib_devs() { - echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu run uath urtwn wpi" + echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu rtwn run uath urtwn wpi" } # These require proprietary firmware that is included in sys/dev -- 2.11.0 From 22e33a832c1eac604359ee60b510d3ab30427d6a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:33:31 +0200 Subject: [PATCH 093/255] sys/conf/newvers.sh: Remove svn support (but keep 'git svn' support) Obtained from: ElectroBSD --- sys/conf/newvers.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index bc688dd63799..27df7f443448 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -116,31 +116,6 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -for dir in /usr/bin /usr/local/bin; do - if [ ! -z "${svnversion}" ] ; then - break - fi - if [ -x "${dir}/svnversion" ] && [ -z ${svnversion} ] ; then - # Run svnversion from ${dir} on this script; if return code - # is not zero, the checkout might not be compatible with the - # svnversion being used. - ${dir}/svnversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=${dir}/svnversion - break - fi - fi -done - -if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then - /usr/bin/svnliteversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=/usr/bin/svnliteversion - else - svnversion= - fi -fi - if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -150,14 +125,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -n "$svnversion" ] ; then - svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` - case "$svn" in - [0-9]*) svn=" r${svn}" ;; - *) unset svn ;; - esac -fi - if [ -n "$git_cmd" ] ; then git=`$git_cmd rev-parse --verify --short HEAD 2>/dev/null` svn=`$git_cmd svn find-rev $git 2>/dev/null` -- 2.11.0 From c55ce8b96f669839cae548ca37d86b36175bf777 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:34:45 +0200 Subject: [PATCH 094/255] sys/conf/newvers.sh: Do not add git hash from .git if KERNEL_VERSION_NUMBER is defined ... as this indicates that we are (trying to) reproduce a build. Do not use REPRO_SEED as it's always set nowadays. XXX: There should be a cleaner way to do this. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 27df7f443448..e6364d4675f2 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -116,7 +116,7 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -if [ -d "${SYSDIR}/../.git" ] ; then +if [ -z "${KERNEL_VERSION_NUMBER}" -a -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then git_cmd="${dir}/git --git-dir=${SYSDIR}/../.git" -- 2.11.0 From e9236d8c836b9da64e0cf0cc2cc2afce8ba40614 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 28 May 2015 15:04:48 +0200 Subject: [PATCH 095/255] sys/conf/newvers.sh: Allow to overwrite the kernel version ... as the object directory may be dirty. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index e6364d4675f2..43a793bb1428 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -97,7 +97,7 @@ then fi touch version -v=`cat version` +v=${KERNEL_VERSION_NUMBER:-`cat version`} u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} @@ -170,4 +170,6 @@ int osreldate = ${RELDATE}; char kern_ident[] = "${i}"; EOF -echo $((v + 1)) > version +if [ -z "${KERNEL_VERSION_NUMBER}" ]; then + echo $((v + 1)) > version +fi -- 2.11.0 From 97f4506d8a5a920250ab296bf2f1874d1b1c981b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Sep 2015 11:53:45 +0200 Subject: [PATCH 096/255] make-memstick.sh: Additionally fake uid and gid on the created fs Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index a5d88e166e23..25c00a501131 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -30,7 +30,12 @@ fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +# Prepare mtree spec to fake timestamp, owner and group. +# As a result, man pages will be owned by root instead of man. +# Unfortunately we can't simply reuse ${1}/METALOG as it is incomplete. +mtree -c -k time -p "${1}" | sed \ + -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ + > "${2}.mtree" || return 1 makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" -- 2.11.0 From afd8f5e16c1ac915dd3069e043b8c2ceb0fc84f7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 30 Sep 2015 13:15:44 +0200 Subject: [PATCH 097/255] release/amd64/make-memstick.sh: Error out if mkimg fails instead of cleaning up Makes debugging more convenient. Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 25c00a501131..a39a200dde22 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -46,5 +46,9 @@ rm ${1}/etc/rc.conf.local rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +if [ $? -ne 0 ]; then + echo "mkimg failed" + exit 1 +fi rm ${2}.part -- 2.11.0 From d610449a1cac1c9458075c3f38890f158181abd3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:43:06 +0100 Subject: [PATCH 098/255] release/amd64/make-memstick.sh: Stop creating a puny swap partion Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index a39a200dde22..226846ff1c8d 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -45,7 +45,7 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.11.0 From a0104e90e8231f44a707394f565315d39333a6df Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:44:10 +0100 Subject: [PATCH 099/255] release/amd64/make-memstick.sh: Stop creating an EFI partition as the EFI goo does not build reproducible Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 226846ff1c8d..3feded5a1440 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -45,7 +45,7 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.11.0 From 15cf764f140fdd34eab8f6ad5ab690b4d6610e56 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 17:26:59 +0200 Subject: [PATCH 100/255] Add reproduce.sh which makes reproducing ElectroBSD more convenient Squashed commits worth mentioning: - Add -j option to overwrite the maximum number of make jobs - Assert that the source directory is untainted (according to strip-freebsd.sh) and add -a flag to remove offending files - Allow to resume a build by using the -r flag. Obtained from: ElectroBSD --- reproduce.sh | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100755 reproduce.sh diff --git a/reproduce.sh b/reproduce.sh new file mode 100755 index 000000000000..d7b4fa941817 --- /dev/null +++ b/reproduce.sh @@ -0,0 +1,196 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# reproduce.sh +# +# Script to make reproducing an ElectroBSD build more convenient. +# Before using it, make sure BUILD and EPOCH contain values other +# than __BUILD__ and __EPOCH__, either by editing the script or +# by putting them in a configuration file that is speficied with +# the -f option. + +# These variables have to be set to the values used for the build +# that is supposed to be reproduced. ${SRC_DIR} must contain the +# matching sources! +BUILD=__BUILD__ +EPOCH=__EPOCH__ + +# This is just a suggestion, feel free to overwrite it with the -j option. +MAX_MAKE_JOBS="${MAX_MAKE_JOBS-4}" + +# Currently hardcoded. +SRC_DIR=/usr/src +OPTIONAL_CONFIG_FILE="${SRC_DIR}/reproduce.conf" + +# Make sure we respawn with the same script, even if it is located +# outside the SRC_DIR and called with a relative path. +REPRODUCE_SH="$(realpath "${0}")" + +# When set to true, existing object files will be reused. +# If the source files changed, the result will not be reproducible! +RESUME_BUILD="${RESUME_BUILD-false}" + +announce_status() { + local msg \ + timestamp + + msg="${*}" + timestamp=$(date "+%Y-%m-%d %H:%M") + + echo "${timestamp}: ${msg}" +} + +reproduce_all_the_things() { + + if "${RESUME_BUILD}"; then + announce_status "Resuming ..." + export KERNFAST=1 + export NO_CLEAN=1 + fi + + announce_status "Starting to build the kernel" + make buildkernel || return 1 + + announce_status "Starting to build the world" + make -j${MAX_MAKE_JOBS} buildworld || return 1 + + # Make sure obj files aren't dumped in ${SRC_DIR} + mkdir -p "/usr/obj${SRC_DIR}/release" || return 1 + + if ! "${RESUME_BUILD}"; then + announce_status "Starting to clean the release dir" + make -C "${SRC_DIR}/release" clean + fi + announce_status "Starting to build the release" + time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 + + announce_status "Done with release memstick for ${REPRO_SEED}" +} + +assert_untainted_source_tree() { + local auto_untaint \ + untaint_commands + + auto_untaint="${1}" + + untaint_commands="$(sh ./release/scripts/strip-freebsd.sh .)" + if [ -n "${untaint_commands}" ] ; then + if $auto_untaint; then + echo "Auto untainting $(pwd)" + echo "${untaint_commands}" | sh -x || return 1 + else + echo "${SRC_DIR} is tainted. Use -a flag to auto-untaint it." + return 1 + fi + fi + return 0 +} + +respawn_with_clean_environment() { + exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ + LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ + MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ + REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" +} + +main() { + local args \ + auto_untaint config_file fake_user dry_run + + fake_user=elektropunker + + auto_untaint=false + dry_run=false + config_file="${OPTIONAL_CONFIG_FILE}" + + args=$(getopt af:j:nr $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -a) + shift + auto_untaint="true" + ;; + -j) + shift; + MAX_MAKE_JOBS="${1}" + shift; + ;; + -f) + shift; + config_file="${1}" + shift; + if [ ! -f "${config_file}" ]; then + echo "Config file ${config_file} does not exist" + exit 2 + fi + ;; + -n) + dry_run=true + shift + ;; + -r) + shift + RESUME_BUILD=true + ;; + --) + shift; break + ;; + esac + done + + if [ -f "${config_file}" ]; then + announce_status "Reading config from ${config_file}" + . "${config_file}" || exit 2 + fi + + if [ "${BUILD}" = "__BUILD__" ]; then + announce_status "BUILD not set" + return 1 + fi + if [ "${EPOCH}" = "__EPOCH__" ]; then + announce_status "EPOCH not set" + return 1 + fi + if [ -n "${ALREADY_RESPAWNED}" -a "${ALREADY_RESPAWNED}" = 1 ]; then + if [ -z "${REPRO_SEED}" ]; then + announce_status "Respawned with REPRO_SEED unset" + return 1 + fi + reproduce_all_the_things + return + fi + + export REPRO_SEED="${fake_user}:${BUILD}:${EPOCH}" + + announce_status "REPRO_SEED=${REPRO_SEED}" + + cd "${SRC_DIR}" || return 1 + + assert_untainted_source_tree "${auto_untaint}" || return 1 + + if ! $dry_run; then + respawn_with_clean_environment + fi +} + +main "${@}" -- 2.11.0 From ce889d11e1ce6f82c2e2057e20033cdc110cd829 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 25 Dec 2015 15:24:18 +0100 Subject: [PATCH 101/255] reproduce.sh: Add -p option to change the prefix for the source and object directories This could be useful when building ElectroBSD as port. Obtained from: ElectroBSD --- reproduce.sh | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index d7b4fa941817..31be55b7ba04 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -33,8 +33,14 @@ EPOCH=__EPOCH__ # This is just a suggestion, feel free to overwrite it with the -j option. MAX_MAKE_JOBS="${MAX_MAKE_JOBS-4}" -# Currently hardcoded. -SRC_DIR=/usr/src +# Prefix to use for SRC_DIR and MAKEOBJDIRPREFIX +DIRECTORY_PREFIX="${DIRECTORY_PREFIX-/}" + +# Currently somewhat hardcoded. +SRC_DIR="${DIRECTORY_PREFIX}usr/src" +MAKEOBJDIRPREFIX="${DIRECTORY_PREFIX}usr/obj" + +# Config file location when -f isn't specified OPTIONAL_CONFIG_FILE="${SRC_DIR}/reproduce.conf" # Make sure we respawn with the same script, even if it is located @@ -62,6 +68,8 @@ reproduce_all_the_things() { export KERNFAST=1 export NO_CLEAN=1 fi + export MAKEOBJDIRPREFIX + announce_status "MAKEOBJDIRPREFIX is set to ${MAKEOBJDIRPREFIX}" announce_status "Starting to build the kernel" make buildkernel || return 1 @@ -70,7 +78,7 @@ reproduce_all_the_things() { make -j${MAX_MAKE_JOBS} buildworld || return 1 # Make sure obj files aren't dumped in ${SRC_DIR} - mkdir -p "/usr/obj${SRC_DIR}/release" || return 1 + mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 if ! "${RESUME_BUILD}"; then announce_status "Starting to clean the release dir" @@ -105,6 +113,7 @@ respawn_with_clean_environment() { exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ + DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" \ REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" } @@ -118,7 +127,7 @@ main() { dry_run=false config_file="${OPTIONAL_CONFIG_FILE}" - args=$(getopt af:j:nr $*) + args=$(getopt af:j:npr $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -148,6 +157,17 @@ main() { dry_run=true shift ;; + -p) + shift + DIRECTORY_PREFIX="${1}" + shift; + if [ ! -d "${DIRECTORY_PREFIX}" ]; then + echo "Directory ${DIRECTORY_PREFIX} specified with -p does not exist" + exit 2 + fi + # This is only needed for the cd below + SRC_DIR="${DIRECTORY_PREFIX}${SRC_DIR}" + ;; -r) shift RESUME_BUILD=true -- 2.11.0 From c97682a9a3811a154f6784a485c9d96885a85542 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 Jan 2016 19:57:05 +0100 Subject: [PATCH 102/255] reproduce.sh: Build kernel after world to make sure the toolchain is fresh Obtained from: ElectroBSD --- reproduce.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index 31be55b7ba04..4783dd510775 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -71,12 +71,14 @@ reproduce_all_the_things() { export MAKEOBJDIRPREFIX announce_status "MAKEOBJDIRPREFIX is set to ${MAKEOBJDIRPREFIX}" - announce_status "Starting to build the kernel" - make buildkernel || return 1 - + # We build the world first, so the kernel is built + # with a freshly built toolchain. announce_status "Starting to build the world" make -j${MAX_MAKE_JOBS} buildworld || return 1 + announce_status "Starting to build the kernel" + make buildkernel || return 1 + # Make sure obj files aren't dumped in ${SRC_DIR} mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 -- 2.11.0 From 306af9fd417bde214c287384309e9d8b3bf75aa4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 15 Jan 2016 18:35:29 +0100 Subject: [PATCH 103/255] reproduce.sh: Move created files to their own directory when done Obtained from: ElectroBSD --- reproduce.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/reproduce.sh b/reproduce.sh index 4783dd510775..125d9fa407a1 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -62,7 +62,10 @@ announce_status() { } reproduce_all_the_things() { + local \ + f release_dir final_dir + release_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/release" if "${RESUME_BUILD}"; then announce_status "Resuming ..." export KERNFAST=1 @@ -80,7 +83,7 @@ reproduce_all_the_things() { make buildkernel || return 1 # Make sure obj files aren't dumped in ${SRC_DIR} - mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 + mkdir -p "${release_dir}" || return 1 if ! "${RESUME_BUILD}"; then announce_status "Starting to clean the release dir" @@ -90,6 +93,18 @@ reproduce_all_the_things() { time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 announce_status "Done with release memstick for ${REPRO_SEED}" + + cd "${release_dir}" || return 1 + + final_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/${BUILD}-$(date "+%Y-%m-%d-%H:%M")" + mkdir "${final_dir}" || return 1 + for f in *.txz MANIFEST; do + mv "${f}" "${final_dir}/" || return + done + mv memstick.img "${final_dir}/${BUILD}.img" || return 1 + + announce_status "Created files copied to ${final_dir}" + sha256 "${final_dir}"/* } assert_untainted_source_tree() { -- 2.11.0 From b452016e70a984dbed4bed490acb4cf46a0135f0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 Jul 2016 12:19:53 +0200 Subject: [PATCH 104/255] reproduce.sh: Hardcode the number of xz threads to 2 Due to some upstream change hardcoding it to something is required now to get reproducible results when using systems with a different number of cores. Obtained from: ElectroBSD --- reproduce.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index 125d9fa407a1..6eb83f54c451 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -51,6 +51,13 @@ REPRODUCE_SH="$(realpath "${0}")" # If the source files changed, the result will not be reproducible! RESUME_BUILD="${RESUME_BUILD-false}" +# Number of threads to use when compressing with xz. +# +# The upstream default is 0 (auto-tune) which results +# in unreproducible results when using systems with a +# different number of cores. +XZ_THREADS="${XZ_THREADS-2}" + announce_status() { local msg \ timestamp @@ -90,7 +97,8 @@ reproduce_all_the_things() { make -C "${SRC_DIR}/release" clean fi announce_status "Starting to build the release" - time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 + time make -C "${SRC_DIR}/release" \ + memstick XZ_THREADS=${XZ_THREADS} NO_FSCHG="yes" || return 1 announce_status "Done with release memstick for ${REPRO_SEED}" @@ -130,7 +138,7 @@ respawn_with_clean_environment() { exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ - DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" \ + DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" XZ_THREADS=${XZ_THREADS} \ REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" } -- 2.11.0 From 9299532ada19462ab3b02546c53e9b37ee1044d9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Nov 2016 15:13:42 +0100 Subject: [PATCH 105/255] reproduce.sh: Add a -d option to specifiy the directory to move the produced distfiles into Obtained from: ElectroBSD --- reproduce.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index 6eb83f54c451..cd5c5cb0a30b 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -1,7 +1,7 @@ #!/bin/sh ########################################################################## -# Copyright (c) 2015 Fabian Keil +# Copyright (c) 2015-2016 Fabian Keil # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -70,7 +70,7 @@ announce_status() { reproduce_all_the_things() { local \ - f release_dir final_dir + f release_dir release_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/release" if "${RESUME_BUILD}"; then @@ -104,15 +104,14 @@ reproduce_all_the_things() { cd "${release_dir}" || return 1 - final_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/${BUILD}-$(date "+%Y-%m-%d-%H:%M")" - mkdir "${final_dir}" || return 1 + mkdir -p "${DISTFILE_DIR}" || return 1 for f in *.txz MANIFEST; do - mv "${f}" "${final_dir}/" || return + mv "${f}" "${DISTFILE_DIR}/" || return done - mv memstick.img "${final_dir}/${BUILD}.img" || return 1 + mv memstick.img "${DISTFILE_DIR}/${BUILD}.img" || return 1 - announce_status "Created files copied to ${final_dir}" - sha256 "${final_dir}"/* + announce_status "Created files copied to ${DISTFILE_DIR}" + sha256 "${DISTFILE_DIR}"/* } assert_untainted_source_tree() { @@ -137,6 +136,7 @@ assert_untainted_source_tree() { respawn_with_clean_environment() { exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ + DISTFILE_DIR="${DISTFILE_DIR}" \ MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" XZ_THREADS=${XZ_THREADS} \ REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" @@ -152,7 +152,7 @@ main() { dry_run=false config_file="${OPTIONAL_CONFIG_FILE}" - args=$(getopt af:j:npr $*) + args=$(getopt ad:f:j:npr $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -164,6 +164,11 @@ main() { shift auto_untaint="true" ;; + -d) + shift; + DISTFILE_DIR="${1}" + shift; + ;; -j) shift; MAX_MAKE_JOBS="${1}" @@ -226,6 +231,7 @@ main() { fi export REPRO_SEED="${fake_user}:${BUILD}:${EPOCH}" + export DISTFILE_DIR="${DISTFILE_DIR-${MAKEOBJDIRPREFIX}${SRC_DIR}/${BUILD}-$(date +%Y-%m-%d-%H:%M)}" announce_status "REPRO_SEED=${REPRO_SEED}" -- 2.11.0 From 461627c97caa2ed9f20a8e850e6747cee595fa25 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Nov 2015 18:55:18 +0100 Subject: [PATCH 106/255] share/doc: Detach 'legal' from the build It (tries to) install proprietary licenses for code that is not part of ElectroBSD and deleted by reproduce.sh's auto-untaint mode (-a). Obtained from: ElectroBSD --- share/doc/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/share/doc/Makefile b/share/doc/Makefile index 7a02b2952598..5b5e507bc130 100644 --- a/share/doc/Makefile +++ b/share/doc/Makefile @@ -5,7 +5,6 @@ SUBDIR= ${_IPv6} \ ${_atf} \ - legal \ ${_llvm} \ ${_pjdfstest} \ ${_roffdocs} -- 2.11.0 From 6b271fef8657b270400bdc2a5129dbd5ee2c2540 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 13:38:21 +0200 Subject: [PATCH 107/255] sys/boot/Makefile.amd64: Detach efi from the built It doesn't built reproducible, among other things due to the embedded file system, and none of the ElectroBSD users I'm aware of have efi-capable systems anyway. Obtained from: ElectroBSD --- sys/boot/Makefile.amd64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/Makefile.amd64 b/sys/boot/Makefile.amd64 index 5e730bd6c62f..f8e173cbb2f3 100644 --- a/sys/boot/Makefile.amd64 +++ b/sys/boot/Makefile.amd64 @@ -1,6 +1,6 @@ # $FreeBSD$ -SUBDIR+= efi +#SUBDIR+= efi SUBDIR+= libstand32 SUBDIR+= zfs SUBDIR+= userboot -- 2.11.0 From 8e3a311a2c0208eaf0ca41f14a0dd6841557bbc6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 May 2015 18:40:56 +0200 Subject: [PATCH 108/255] release/amd64/make-memstick.sh: Allow to overwrite VOLUME_LABEL through the environment Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 3feded5a1440..66ee073c63ba 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -12,6 +12,7 @@ PATH=/bin:/usr/bin:/sbin:/usr/sbin export PATH +VOLUME_LABEL=${VOLUME_LABEL-"FreeBSD_Install"} if [ $# -ne 2 ]; then echo "make-memstick.sh /path/to/directory /path/to/image/file" @@ -28,7 +29,7 @@ if [ -e ${2} ]; then exit 1 fi -echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab +echo "/dev/ufs/${VOLUME_LABEL} / ufs ro,noatime 1 1" > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local # Prepare mtree spec to fake timestamp, owner and group. # As a result, man pages will be owned by root instead of man. @@ -36,7 +37,7 @@ echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local mtree -c -k time -p "${1}" | sed \ -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ > "${2}.mtree" || return 1 -makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} +makefs -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 -- 2.11.0 From 3643c5dd889e79de5b0b5078954f52a7cfc98fcf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 May 2015 18:42:42 +0200 Subject: [PATCH 109/255] release/Makefile: Export VOLUME_LABEL for make-memstick.sh Adding the UNAME_s to it probably would not hurt ... Obtained from: ElectroBSD --- release/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index 83d46a499b55..632e4708fad5 100644 --- a/release/Makefile +++ b/release/Makefile @@ -77,9 +77,11 @@ VOLUME_LABEL= ${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET_ARCH} .endif .if !defined(VOLUME_LABEL) || empty(VOLUME_LABEL) -VOLUME_LABEL= FreeBSD_Install +VOLUME_LABEL= FreeBSD_Install .endif +.export VOLUME_LABEL + .if !exists(${DOCDIR}) NODOC= true .endif -- 2.11.0 From 06bf0d0bad39b0babe1447ed9c4405ba5c93de2f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 3 Dec 2015 18:34:21 +0100 Subject: [PATCH 110/255] lib/libkvm: Unbreak the build without kernel sources that don't ship with ElectroBSD Obtained from: ElectroBSD --- lib/libkvm/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/libkvm/Makefile b/lib/libkvm/Makefile index dc611a4b4bfe..8a6806102797 100644 --- a/lib/libkvm/Makefile +++ b/lib/libkvm/Makefile @@ -12,13 +12,8 @@ WARNS?= 3 SRCS= kvm.c kvm_cptime.c kvm_getloadavg.c \ kvm_getswapinfo.c kvm_pcpu.c kvm_proc.c kvm_vnet.c \ - kvm_minidump_aarch64.c \ kvm_amd64.c kvm_minidump_amd64.c \ - kvm_arm.c kvm_minidump_arm.c \ - kvm_i386.c kvm_minidump_i386.c \ - kvm_minidump_mips.c \ - kvm_powerpc.c kvm_powerpc64.c \ - kvm_sparc64.c + kvm_i386.c kvm_minidump_i386.c INCS= kvm.h LIBADD= elf -- 2.11.0 From 83b4c902b220fc8193af4f8ab11ea68210566012 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 May 2015 14:03:06 +0200 Subject: [PATCH 111/255] usr.sbin/pkg: Change URL_SCHEME_PREFIX to an URL more easily controlled by the user ... through ssh port forwarding. Obtained from: ElectroBSD --- usr.sbin/pkg/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.sbin/pkg/config.c b/usr.sbin/pkg/config.c index 2fea11f50fe4..8cc35190054f 100644 --- a/usr.sbin/pkg/config.c +++ b/usr.sbin/pkg/config.c @@ -64,7 +64,7 @@ static struct config_entry c[] = { [PACKAGESITE] = { PKG_CONFIG_STRING, "PACKAGESITE", - URL_SCHEME_PREFIX "http://pkg.FreeBSD.org/${ABI}/latest", + URL_SCHEME_PREFIX "http://127.0.0.1:8000/packages/${ABI}/", NULL, NULL, false, -- 2.11.0 From ae683e2bf95b912a8daf753c56e1942ced11b055 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 15:33:59 +0200 Subject: [PATCH 112/255] share/doc: Exclude directories that contain (partly) non-free materials Obtained from: ElectroBSD --- share/doc/Makefile | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/share/doc/Makefile b/share/doc/Makefile index 5b5e507bc130..dc2003f7ecfe 100644 --- a/share/doc/Makefile +++ b/share/doc/Makefile @@ -3,11 +3,9 @@ .include -SUBDIR= ${_IPv6} \ - ${_atf} \ +SUBDIR= ${_atf} \ ${_llvm} \ - ${_pjdfstest} \ - ${_roffdocs} + ${_pjdfstest} .if ${MK_TESTS} != "no" _atf= atf @@ -18,14 +16,11 @@ _pjdfstest= pjdfstest _llvm= llvm .endif -.if ${MK_INET6} != "no" -_IPv6= IPv6 -.endif - -# FIXME this is not a real solution ... -.if ${MK_GROFF} != "no" -_roffdocs= papers psd smm usd -.endif +# Note: Documents in the sub directories IPv6, papers, psd, smm +# and usd are not installed on ElectroBSD due to license problems. +# +# Some of the excluded papers actually have free licenses and +# should eventually be installed. SUBDIR_PARALLEL= -- 2.11.0 From d9b1c3433ce69d0cfacc0a77249b3aad0e650037 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 10:10:43 +0200 Subject: [PATCH 113/255] geom: Do not build raid, raid3 and vinum classes Obtained from: ElectroBSD --- sys/modules/geom/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/sys/modules/geom/Makefile b/sys/modules/geom/Makefile index 8d7e3c6deb3f..fe7e479840aa 100644 --- a/sys/modules/geom/Makefile +++ b/sys/modules/geom/Makefile @@ -16,13 +16,10 @@ SUBDIR= geom_bde \ geom_multipath \ geom_nop \ geom_part \ - geom_raid \ - geom_raid3 \ geom_sched \ geom_shsec \ geom_stripe \ geom_uzip \ - geom_vinum \ geom_virstor \ geom_zero -- 2.11.0 From 7f1d7c4389d7ce7131a496eeb22b2c86ae8f73e3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 12:17:36 +0200 Subject: [PATCH 114/255] share/mk/src.opts.mk: Disable a bunch of options by default .. either due to license issues or because they are not considered relevant for the majority of ElectroBSD users. XXX: This commit does not regenerate src.conf(5) as the process hasn't been reverse-engineered yet. Obtained from: ElectroBSD --- share/mk/src.opts.mk | 58 ++++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 770d8b6fc066..57c70a61ade9 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -49,7 +49,6 @@ __DEFAULT_YES_OPTIONS = \ AMD \ APM \ AT \ - ATM \ AUDIT \ AUTHPF \ AUTOFS \ @@ -57,23 +56,19 @@ __DEFAULT_YES_OPTIONS = \ BINUTILS \ BINUTILS_BOOTSTRAP \ BLACKLIST \ - BLUETOOTH \ BOOT \ BOOTPARAMD \ BOOTPD \ BSD_CPIO \ - BSDINSTALL \ BSNMP \ BZIP2 \ CALENDAR \ CAPSICUM \ CASPER \ - CCD \ CDDL \ CPP \ CROSS_COMPILER \ CRYPT \ - CTM \ CUSE \ CXX \ DICT \ @@ -87,14 +82,9 @@ __DEFAULT_YES_OPTIONS = \ FDT \ FILE \ FINGER \ - FLOPPY \ - FMTREE \ FORTH \ FP_LIBC \ - FREEBSD_UPDATE \ - FTP \ GAMES \ - GCOV \ GDB \ GNU \ GNU_GREP_COMPAT \ @@ -108,9 +98,7 @@ __DEFAULT_YES_OPTIONS = \ INET \ INET6 \ INETD \ - IPFILTER \ IPFW \ - ISCSI \ JAIL \ KDUMP \ KVM \ @@ -129,7 +117,6 @@ __DEFAULT_YES_OPTIONS = \ MAILWRAPPER \ MAKE \ MANDOCDB \ - NDIS \ NETCAT \ NETGRAPH \ NLS_CATALOGS \ @@ -137,36 +124,22 @@ __DEFAULT_YES_OPTIONS = \ NTP \ OPENSSL \ PAM \ - PC_SYSINSTALL \ PF \ PKGBOOTSTRAP \ - PMC \ - PORTSNAP \ - PPP \ QUOTAS \ RADIUS_SUPPORT \ - RCMDS \ - RBOOTD \ RCS \ RESCUE \ ROUTED \ - SENDMAIL \ SETUID_LOGIN \ SHAREDOCS \ - SOURCELESS \ - SOURCELESS_HOST \ - SOURCELESS_UCODE \ - SVNLITE \ SYSCONS \ SYSTEM_COMPILER \ TALK \ - TCP_WRAPPERS \ - TCSH \ TELNET \ TESTS \ TEXTPROC \ TFTP \ - TIMED \ UNBOUND \ USB \ UTMPX \ @@ -192,6 +165,37 @@ __DEFAULT_NO_OPTIONS = \ SVN \ +# Disable a bunch of additional options that default to yes in FreeBSD +__DEFAULT_NO_OPTIONS += \ + ATM \ + BLUETOOTH \ + BSDINSTALL \ + CCD \ + CTM \ + FLOPPY \ + FMTREE \ + FREEBSD_UPDATE \ + FTP \ + GCOV \ + IPFILTER \ + ISCSI \ + NDIS \ + RBOOTD \ + PC_SYSINSTALL \ + PMC \ + PORTSNAP \ + PPP \ + RCMDS \ + SENDMAIL \ + SVNLITE \ + SOURCELESS \ + SOURCELESS_HOST \ + SOURCELESS_UCODE \ + SYSINSTALL \ + TCP_WRAPPERS \ + TCSH \ + TIMED \ + # # Default behaviour of some options depends on the architecture. Unfortunately # this means that we have to test TARGET_ARCH (the buildworld case) as well -- 2.11.0 From b94ee129bbe69f75d95b6c8237f4306d3f47c54d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 24 Jun 2015 12:53:20 +0200 Subject: [PATCH 115/255] share/mk/src.opts.mk: Enable CLANG_EXTRAS by default as it's required for llvm-symbolizer Obtained from: ElectroBSD --- share/mk/src.opts.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 57c70a61ade9..d71083d52e77 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -152,7 +152,6 @@ __DEFAULT_YES_OPTIONS = \ __DEFAULT_NO_OPTIONS = \ BSD_GREP \ - CLANG_EXTRAS \ DTRACE_TESTS \ EISA \ HESIOD \ @@ -165,6 +164,10 @@ __DEFAULT_NO_OPTIONS = \ SVN \ +# Enable additional options that default to NO in FreeBSD +__DEFAULT_YES_OPTIONS += \ + CLANG_EXTRAS \ + # Disable a bunch of additional options that default to yes in FreeBSD __DEFAULT_NO_OPTIONS += \ ATM \ -- 2.11.0 From c8732639120de730d42c3655647c16c7586666c8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 17:53:19 +0200 Subject: [PATCH 116/255] etc/master.passwd: Change root's login shell to sh Now that csh is no longer compiled by default that seems like a rather swell idea. Obtained from: ElectroBSD --- etc/master.passwd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/master.passwd b/etc/master.passwd index 689b64e78473..c6005dc66899 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -1,6 +1,6 @@ # $FreeBSD$ # -root::0:0::0:0:Charlie &:/root:/bin/csh +root::0:0::0:0:Charlie &:/root:/bin/sh toor:*:0:0::0:0:Bourne-again Superuser:/root: daemon:*:1:1::0:0:Owner of many system processes:/root:/usr/sbin/nologin operator:*:2:5::0:0:System &:/:/usr/sbin/nologin -- 2.11.0 From 651497e96a360f083e22dc898fe8a646fb28a539 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 12:03:46 +0200 Subject: [PATCH 117/255] release/Makefile: Set German keyboard map Obtained from: ElectroBSD --- release/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/release/Makefile b/release/Makefile index 632e4708fad5..ff114ce6d28d 100644 --- a/release/Makefile +++ b/release/Makefile @@ -216,6 +216,7 @@ disc1: packagesystem ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf + echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.11.0 From 7e65b183a886b524b1076cb2f4b874aaf9ed8bdc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 12:06:14 +0200 Subject: [PATCH 118/255] release/Makefile: Set hostname to ${VOLUME_LABEL} Obtained from: ElectroBSD --- release/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/release/Makefile b/release/Makefile index ff114ce6d28d..18e4cb817be1 100644 --- a/release/Makefile +++ b/release/Makefile @@ -217,6 +217,7 @@ disc1: packagesystem echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf + echo hostname=\"${VOLUME_LABEL}\" >> ${.TARGET}/etc/rc.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.11.0 From a55e4121b649ccf39d7825502feca66925167c80 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 19:38:39 +0200 Subject: [PATCH 119/255] Stop bothering packaging docs which are no longer built. Squash or relocate Obtained from: ElectroBSD --- Makefile.inc1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 4b6c8d77834f..6358d477d44d 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -925,7 +925,7 @@ ITOOLS+=makewhatis # # Non-base distributions produced by the base system -EXTRA_DISTRIBUTIONS= doc +EXTRA_DISTRIBUTIONS= .if defined(LIBCOMPAT) EXTRA_DISTRIBUTIONS+= lib${libcompat} .endif -- 2.11.0 From 74032f9724c7a8d290a438baf006bd2a99573458 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 10 Jun 2015 12:29:37 +0200 Subject: [PATCH 120/255] release/Makefile: Add ${DIST_TARBALL_DIR} ... which contains the distribution tarball directory on the install media. Obtained from: ElectroBSD --- release/Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/release/Makefile b/release/Makefile index 18e4cb817be1..bc435fa8defb 100644 --- a/release/Makefile +++ b/release/Makefile @@ -60,6 +60,9 @@ DISTDIR= dist NO_ROOT=1 .export NO_ROOT +# Path to the distribution tarballs on the created installation media. +DIST_TARBALL_DIR=usr/freebsd-dist + # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -204,9 +207,9 @@ disc1: packagesystem MK_DEBUG_FILES=no MK_LLDB=no \ MK_TOOLCHAIN=no # Copy distfiles - mkdir -p ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} for dist in MANIFEST $$(ls *.txz | grep -vE -- '(base|lib32|kernel)-dbg'); \ - do cp $${dist} ${.TARGET}/usr/freebsd-dist; \ + do cp $${dist} ${.TARGET}/${DIST_TARBALL_DIR}; \ done # Copy documentation, if generated .if !defined(NODOC) @@ -233,8 +236,8 @@ bootonly: packagesystem MK_RESCUE=no MK_DICT=no \ MK_KERNEL_SYMBOLS=no MK_TESTS=no MK_DEBUG_FILES=no # Copy manifest only (no distfiles) to get checksums - mkdir -p ${.TARGET}/usr/freebsd-dist - cp MANIFEST ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} + cp MANIFEST ${.TARGET}/${DIST_TARBALL_DIR} # Copy documentation, if generated .if !defined(NODOC) cp reldoc/* ${.TARGET} @@ -253,9 +256,9 @@ dvd: packagesystem DESTDIR=${.OBJDIR}/${.TARGET} MK_RESCUE=no MK_KERNEL_SYMBOLS=no \ MK_TESTS=no MK_DEBUG_FILES=no # Copy distfiles - mkdir -p ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} for dist in MANIFEST $$(ls *.txz | grep -v -- '(base|lib32)-dbg'); \ - do cp $${dist} ${.TARGET}/usr/freebsd-dist; \ + do cp $${dist} ${.TARGET}/${DIST_TARBALL_DIR}; \ done # Copy documentation, if generated .if !defined(NODOC) -- 2.11.0 From 8c2e7dd1c1c8cbe8f4453fd04ea3c0c312cc5942 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 10 Jun 2015 12:36:37 +0200 Subject: [PATCH 121/255] release/Makefile: ElectroBSDify the shiny new ${DIST_TARBALL_DIR} Obtained from: ElectroBSD --- release/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index bc435fa8defb..baf50a240876 100644 --- a/release/Makefile +++ b/release/Makefile @@ -61,7 +61,7 @@ NO_ROOT=1 .export NO_ROOT # Path to the distribution tarballs on the created installation media. -DIST_TARBALL_DIR=usr/freebsd-dist +DIST_TARBALL_DIR=usr/electrobsd-dist # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) -- 2.11.0 From e771e0ef29d04d74e49e5c9779ba2a52a04a5ae5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 15 Jun 2015 12:57:51 +0200 Subject: [PATCH 122/255] Add rc.d script to automatically enable soft-protection on boot Obtained from: ElectroBSD --- etc/rc.d/Makefile | 1 + etc/rc.d/soft-protection | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 etc/rc.d/soft-protection diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 320e550aeb8a..ec3ad74a4bae 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -104,6 +104,7 @@ FILES= DAEMON \ savecore \ securelevel \ serial \ + soft-protection \ sppp \ statd \ static_arp \ diff --git a/etc/rc.d/soft-protection b/etc/rc.d/soft-protection new file mode 100755 index 000000000000..58bc02a0c037 --- /dev/null +++ b/etc/rc.d/soft-protection @@ -0,0 +1,46 @@ +#!/bin/sh +# +########################################################################### +# +# soft-protection - Enables cloudiatr soft protection on boot +# +########################################################################### +# +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# PROVIDE: soft_protection +# REQUIRE: FILESYSTEMS + +. /etc/rc.subr + +name="soft_protection" +rcvar="soft_protection_enable" + +soft_protection_enable="${soft_protection_enable-NO}" + +start_cmd="enable_soft_protection" +stop_cmd=":" + +enable_soft_protection() { + # We don't use the soft-protect subcommand + # because it may require user feedback. + cloudiatr cmd cloudiatr_soft_protect +} + +load_rc_config "${name}" +run_rc_command "${1}" -- 2.11.0 From 20e6293f3867689451153b9c96af1ed9ad739014 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 16 Jun 2015 15:21:46 +0200 Subject: [PATCH 123/255] Import geli-key-monitor Obtained from: ElectroBSD --- share/dtrace/Makefile | 1 + share/dtrace/README | 3 +- share/dtrace/geli-key-monitor | 191 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100755 share/dtrace/geli-key-monitor diff --git a/share/dtrace/Makefile b/share/dtrace/Makefile index efd8b08706ba..742fa6ea1ab7 100644 --- a/share/dtrace/Makefile +++ b/share/dtrace/Makefile @@ -11,6 +11,7 @@ SCRIPTS= blocking \ disklatencycmd \ hotopen \ nfsattrstats \ + geli-key-monitor \ nfsclienttime \ siftr \ tcpconn \ diff --git a/share/dtrace/README b/share/dtrace/README index 6855c6aee721..aeb2a06727b5 100644 --- a/share/dtrace/README +++ b/share/dtrace/README @@ -2,4 +2,5 @@ $FreeBSD$ This directory contains scripts for use with the DTrace system. These files and directories contain code generated by the FreeBSD -Project for use with DTrace on FreeBSD. +Project for use with DTrace on FreeBSD and code generated by the +ElectroBSD project for use with DTrace on ElectroBSD. diff --git a/share/dtrace/geli-key-monitor b/share/dtrace/geli-key-monitor new file mode 100755 index 000000000000..2cf4ba5c09be --- /dev/null +++ b/share/dtrace/geli-key-monitor @@ -0,0 +1,191 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * geli-key-monitor + * + * Traces GELI to print the beginning of various keys and warns + * about already known ones. For motivation and example output see: + * https://www.fabiankeil.de/gehacktes/geli-key-monitor/ + * + * Copyright (c) 2012 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + algo[ 2] = "3DES-CBC"; + algo[ 3] = "Blowfish-CBC"; + algo[11] = "AES-CBC"; + algo[21] = "Camellia-CBC"; + algo[22] = "AES-XTS"; + + /* + * These are the first bytes of known weak keys generated with + * an unitialized sc->sc_ekey on a little-endian system. + */ + known_keys[0x06d800ef] = 1; + known_keys[0x252c6a6d] = 1; + known_keys[0xa72b9c7c] = 1; + known_keys[0xfde44683] = 1; + + G_ELI_FLAG_ENC_IVKEY = 0x00400000; + + g_eli_key_stats_available = 0; + g_eli_hold_stats_available = 0; + + /* + * Geli v5 and higher use a different key for each GB, + * so large disks have several thousand keys. + * Monitoring only the first few should do, keeps the + * output and reduces the likelihood of prevents dynvar + * drops. + */ + max_disk_keys_to_monitor = 4; + + printf("%Y: Monitoring geli keys (up to %d encryption keys per disk). %s\n", + walltimestamp, max_disk_keys_to_monitor, "Press CTRL-C to exit."); +} + +fbt::g_eli_mkey_propagate:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->mkey = (struct g_eli_key *)arg1; + + self->geom_name = stringof(self->sc->sc_geom->name); + self->monitored_keys = 0; + self->key_limit_reached = 0; + + this->algo = self->sc->sc_ealgo; + this->algo_name = (algo[this->algo] != NULL) ? algo[this->algo] : "Unregistered"; + this->geli_version = self->sc->sc_version; + + printf("%Y: %s: %s:%-6s: version: %d, algo: %s, flags: 0x%x (FLAG_ENC_IVKEY: %d) ", + walltimestamp, + self->geom_name, + probefunc, probename, + this->geli_version, this->algo_name, + self->sc->sc_flags, + (self->sc->sc_flags & G_ELI_FLAG_ENC_IVKEY) != 0 + ); + printf("mkey: %08.8x, sc_mkey: %08.8x, sc_ekey: %08.8x\n", + *(uint32_t *)(self->mkey), + *(uint32_t *)(self->sc->sc_mkey), + *(uint32_t *)(self->sc->sc_ekey)); +} + +fbt::g_eli_mkey_propagate:return +/self->sc != NULL/ +{ + printf("%Y: %s: %s:%-6s: mkey: %08.8x, sc_mkey: %08.8x, sc_ekey: %08.8x, sc_ekeys_allocated: %d\n", + walltimestamp, + self->geom_name, + probefunc, probename, + *(uint32_t *)(self->mkey), + *(uint32_t *)(self->sc->sc_mkey), + *(uint32_t *)(self->sc->sc_ekey), + self->sc->sc_ekeys_allocated); +} + +fbt::g_eli_key_fill:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->key = (struct g_eli_key *)arg1; + self->geom_name = stringof(self->sc->sc_geom->name); +} + +fbt::g_eli_key_fill: +/(self->sc != NULL) && !self->key_limit_reached/ +{ + this->key = *(uint32_t *)(self->key->gek_key); + printf("%Y: %s: %s:%-6s: key->gek_key: %8.8x (%d).%s\n", + walltimestamp, + self->geom_name, + probefunc, probename, + this->key, + self->monitored_keys, + known_keys[this->key] ? " Key looks familiar!" : ""); +} + +fbt::g_eli_key_fill:return +/(self->sc != NULL) && !self->key_limit_reached/ +{ + this->key = *(uint32_t *)(self->key->gek_key); + @g_eli_keys[self->monitored_keys, this->key, self->geom_name] = count(); + /* + * Register the generated key as known. + * Unless the provider is reattached we do not want to see it again. + */ + known_keys[this->key] = 1; + g_eli_key_stats_available = 1; + self->monitored_keys++; +} + +fbt::g_eli_key_fill:return +/(self->sc != NULL) && (self->sc->sc_ekeys_allocated == max_disk_keys_to_monitor - 1)/ +{ + printf("%Y: %s: %s:%-6s: Encryption key limit per disk reached.\n", + walltimestamp, self->geom_name, probefunc, probename); + self->key_limit_reached = 1; +} + +fbt::g_eli_key_hold:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->offset = (off_t)arg1; + self->blocksize = (size_t)arg2; + self->geom_name = stringof(self->sc->sc_geom->name); + + this->first_mkey_bytes = *(uint32_t *)(self->sc->sc_mkey); + this->first_ekey_bytes = *(uint32_t *)(self->sc->sc_ekey); + /* + * We only check the first bytes here, so false-positives are + * theoretically posible although unlikely. + */ + this->empty_ekey = (0 == this->first_ekey_bytes); + + this->geli_version = self->sc->sc_version; + this->algo = self->sc->sc_ealgo; + this->algo_name = (algo[this->algo] != NULL) ? algo[this->algo] : "Unregistered"; + + @g_eli_key_hold[self->geom_name, + this->geli_version, + this->algo_name, + this->algo, + this->first_mkey_bytes, + this->first_ekey_bytes] = count(); + + g_eli_hold_stats_available = 1; +} + +tick-60sec, +dtrace:::END +/g_eli_hold_stats_available/ +{ + printf("\n---------------------------------------------------------\n"); + printf("%Y: g_eli_key_hold() calls so far:\n", walltimestamp); + printf("%-20s %12s %18s %11s %11s %10s\n", + "Provider", "Geli version", "Algorithm", "mkey start", "ekey start", "calls"); + printa("%-20s %12d %13s (%2d) %08x %08x %@10d\n", @g_eli_key_hold); +} + +tick-60sec, +dtrace:::END +/g_eli_key_stats_available/ +{ + printf("\n%Y: g_eli_key count (only works for geli version 5 or higher):\n", walltimestamp); + printa("%@u #%04d %8.8x %-25s\n", @g_eli_keys); +} -- 2.11.0 From 5903e4ace06e0c590b1183f74cb1f24648a84c18 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 18 Jun 2015 12:52:20 +0200 Subject: [PATCH 124/255] jemalloc: Enable MALLOC_PRODUCTION Obtained from: ElectroBSD --- contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h b/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h index e2ddfd8b9c1c..750d669ca27a 100644 --- a/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h +++ b/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h @@ -4,6 +4,8 @@ #undef JEMALLOC_OVERRIDE_VALLOC +#define MALLOC_PRODUCTION + #ifndef MALLOC_PRODUCTION #define MALLOC_PRODUCTION #endif -- 2.11.0 From c66124cfbb3361b73d9c1c2bb3f64216111d2ee0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 16:46:35 +0200 Subject: [PATCH 125/255] release/Makefile: Note that the release process may silently fail Obtained from: ElectroBSD --- release/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release/Makefile b/release/Makefile index baf50a240876..f1fdcdaf7e3d 100644 --- a/release/Makefile +++ b/release/Makefile @@ -2,6 +2,14 @@ # # Makefile for building releases and release media. # +# Note that the release process is rather fragile and lots +# of variables depend on each other in mysterious and +# undocumented ways. +# +# If you don't get the magic right you should consider yourself +# lucky if the build fails, the usual outcome is a release that +# does not work. +# # User-driven targets: # cdrom: Builds release CD-ROM media (disc1.iso) # dvdrom: Builds release DVD-ROM media (dvd1.iso) -- 2.11.0 From 224d301cc83ce6b98e1dd7f529359aa26ef98abd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 16:50:54 +0200 Subject: [PATCH 126/255] release/Makefile: Remove more stuff that does not apply to ElecroBSD from the disc1 target Obtained from: ElectroBSD --- release/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index f1fdcdaf7e3d..27d47434c416 100644 --- a/release/Makefile +++ b/release/Makefile @@ -224,7 +224,6 @@ disc1: packagesystem cp reldoc/* ${.TARGET} .endif # Set up installation environment - ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf -- 2.11.0 From 6cf03c76a0ebe05ce99e416e8bc9e6608c86a0d9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 17:08:38 +0200 Subject: [PATCH 127/255] Makefile.inc1: Workaround passwd and friends missing in the base.txz Obtained from: ElectroBSD --- Makefile.inc1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index 6358d477d44d..33904d920c69 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1030,6 +1030,9 @@ distributeworld installworld stageworld: _installcheck_world .PHONY -e 's@(type=)@uid=0 gid=0 time=${EPOCH_DATE}.0 \1@' \ -e 's@//@/@' -i '.bak' ${METALOG} .endif + @# Workaround for parts of etc mysteriously not being added below base. + @# As usual the location in dist is fine. + sed -e 's@^\./etc@./base/etc@' -i '.etc.bak' ${METALOG} .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that -- 2.11.0 From 3c8e36721d899960772616a28997d364af3c3875 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 27 Jun 2015 19:16:04 +0200 Subject: [PATCH 128/255] ata_da: Ditch support for legacy device names ... and remove the annoying "Previously was known as adX" messages. (The code no longer matches the commit message precisely as upstream had a similar idea a while later and just missed a block which prevented the commit from comletely disappearing while rebasing) Obtained from: ElectroBSD --- sys/cam/ata/ata_da.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c index 808dc72afcb3..6bcc3dba8632 100644 --- a/sys/cam/ata/ata_da.c +++ b/sys/cam/ata/ata_da.c @@ -758,10 +758,6 @@ static void adashutdown(void *arg, int howto); static void adasuspend(void *arg); static void adaresume(void *arg); -#ifndef ADA_DEFAULT_LEGACY_ALIASES -#define ADA_DEFAULT_LEGACY_ALIASES 1 -#endif - #ifndef ADA_DEFAULT_TIMEOUT #define ADA_DEFAULT_TIMEOUT 30 /* Timeout in seconds */ #endif -- 2.11.0 From 7080b8a82ea393cc99bc58c3740b290e832b1703 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 13:45:09 +0200 Subject: [PATCH 129/255] Use make-memstick.sh's amd64 version on i386 as well Obtained from: ElectroBSD --- release/i386/make-memstick.sh | 44 +------------------------------------------ 1 file changed, 1 insertion(+), 43 deletions(-) mode change 100755 => 120000 release/i386/make-memstick.sh diff --git a/release/i386/make-memstick.sh b/release/i386/make-memstick.sh deleted file mode 100755 index f824382ec4b0..000000000000 --- a/release/i386/make-memstick.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# This script generates a "memstick image" (image that can be copied to a -# USB memory stick) from a directory tree. Note that the script does not -# clean up after itself very well for error conditions on purpose so the -# problem can be diagnosed (full filesystem most likely but ...). -# -# Usage: make-memstick.sh -# -# $FreeBSD$ -# - -PATH=/bin:/usr/bin:/sbin:/usr/sbin -export PATH - -if [ $# -ne 2 ]; then - echo "make-memstick.sh /path/to/directory /path/to/image/file" - exit 1 -fi - -if [ ! -d ${1} ]; then - echo "${1} must be a directory" - exit 1 -fi - -if [ -e ${2} ]; then - echo "won't overwrite ${2}" - exit 1 -fi - -echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab -echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -makefs -B little -o label=FreeBSD_Install ${2}.part ${1} -if [ $? -ne 0 ]; then - echo "makefs failed" - exit 1 -fi -rm ${1}/etc/fstab -rm ${1}/etc/rc.conf.local - -mkimg -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} -rm ${2}.part - diff --git a/release/i386/make-memstick.sh b/release/i386/make-memstick.sh new file mode 120000 index 000000000000..978e04a02184 --- /dev/null +++ b/release/i386/make-memstick.sh @@ -0,0 +1 @@ +../amd64/make-memstick.sh \ No newline at end of file -- 2.11.0 From cd36b877da93ff15ba0e51b38d221a3ec6a21d2b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 09:50:52 +0200 Subject: [PATCH 130/255] copyright.h: Use more insightful trademark information Obtained from: ElectroBSD --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index ab9093d76694..0e070c1af60c 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -38,7 +38,7 @@ /* Foundation */ #define TRADEMARK_Foundation \ - "FreeBSD is a registered trademark of The FreeBSD Foundation.\n" + "ElectroBSD ain't no registered trademark of The ElectroBSD Foundation (which does not exist).\n" /* Berkeley */ #define COPYRIGHT_UCB \ -- 2.11.0 From 4e86ce6f6668ee61de95d8c4b62d1e707cc1758f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 09:54:11 +0200 Subject: [PATCH 131/255] g_new_provider_event(): Return early if the provider is already withering Previously debug kernels would panic, other side effects haven't been diagnosed yet. Patch by Scott M. Ferris. FreeBSD bug: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=200740 Obtained from: ElectroBSD --- sys/geom/geom_subr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/geom/geom_subr.c b/sys/geom/geom_subr.c index d0df840945c4..7d19a6fbee15 100644 --- a/sys/geom/geom_subr.c +++ b/sys/geom/geom_subr.c @@ -531,8 +531,8 @@ g_new_provider_event(void *arg, int flag) return; pp = arg; G_VALID_PROVIDER(pp); - KASSERT(!(pp->flags & G_PF_WITHER), - ("g_new_provider_event but withered")); + if ((pp->flags & G_PF_WITHER) != 0) + return; LIST_FOREACH_SAFE(cp, &pp->consumers, consumers, next_cp) { if ((cp->flags & G_CF_ORPHAN) == 0 && cp->geom->attrchanged != NULL) -- 2.11.0 From 68747667eea664825626e9db2127d77bf84581d8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 8 Jul 2015 17:43:31 +0200 Subject: [PATCH 132/255] sys/kern: Remove another FreeBSD reference from the boot messages Obtained from: ElectroBSD --- sys/kern/subr_smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 2d1b8be37aaa..c047115feae3 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -157,7 +157,7 @@ mp_start(void *dummy) } cpu_mp_start(); - printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", + printf("SMP: Multiprocessor System Detected: %d CPUs\n", mp_ncpus); cpu_mp_announce(); } -- 2.11.0 From 92196dfc4af5ec6df5f6b433532474e04bc935c5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 12 Jul 2015 13:11:28 +0200 Subject: [PATCH 133/255] tools/test/devrandom: Import arctest d9a5fc80, a wrapper around dieharder Obtained from: ElectroBSD --- tools/test/devrandom/arc4test | 278 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100755 tools/test/devrandom/arc4test diff --git a/tools/test/devrandom/arc4test b/tools/test/devrandom/arc4test new file mode 100755 index 000000000000..c824997a4a38 --- /dev/null +++ b/tools/test/devrandom/arc4test @@ -0,0 +1,278 @@ +#!/bin/sh + +############################################################################ +# arc4test +# +# Collects "entropy" and lets dieharder analyze it later on. The collected +# entropy is split into smaller files so the data collected in multiple +# runs can be easily interleaved and tested together. +# +# By default, entropy files are generated with arc4cat, a wrapper around +# arc4random_buf(3) which is suspected of "not returning very random data" +# between FreeBSD r273872 and r278907. +# +# So far it looks like the data may be "random enough" to pass the tests. +# +# Usage: +# arc4test build : Build arc4cat in $ARC4CAT_DIR +# arc4test collect : Collect potential entropy with arc4cat +# arc4test collect -d : Collect potential entropy with Dilbert PNRG +# arc4test analyze : Interleave collected entropy files and +# pipe them into dieharder. +# arc4test analyze -f : Try to spead up things by caching the interleaved +# entropy in a single file. Reuses the file if it +# already exists. +# arc4test remix : (Re)build an entropy cache file based on the +# previously collected entropy files. Roughfly +# doubles the required disk space but may significantly +# improve performance. +# arc4test cat : Dump interleaved entropy files to stdout +# +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +############################################################################ + +ARC4CAT_DIR=. +ARC4CAT="${ARC4CAT_DIR}/arc4cat" +# May not contain spaces etc. +ENTROPY_DIR="./entropy" +ENTROPY_SUBDIR_PREFIX="collection-" +# This is currently the block size for dd, setting it "too high" +# is not expected to work, however that's not good for input mixing +# later on anyway. +# +# Currently the entropy is split while it's being collected which is +# convenient from a programming point of view, but if the file size +# is small the interleave performance "may" (vulgo: will) suck. +ENTROPY_FILE_SIZE=4k +# Number of entropy files collected per run. If you increase ENTROPY_FILE_SIZE +# (or intend to do lots of collection runs) you may want to decrease this. +MAX_ENTROPY_FILES=100000 +ENTROPY_CACHE_FILE="${ENTROPY_DIR}/cached-entropy" + +prepare() { +} + +# The default entropy source +get_arc4cat_entropy() { + "${ARC4CAT}" +} + +# This is the reverse engineered PRNG from Dilbert strip 2001-10-25: +# http://dilbert.com/strip/2001-10-25 +# +# It is used instead of get_arc4cat_entropy() if the +# collect flag -d is set. +# +# According to the literature (see URL above) you can never be sure +# if the output is random, however the generator seems to fail all +# the dieharder tests and thus doesn't look nearly as good as Yarrow +# and Fortuna. +# +# Until this changes it will not be considered for ElectroBSD. +get_dilbert_entropy() { + while true; do + # The loop has been partially unrolled + # for increased performance! + echo -n "999999" + done +} + +get_shiny_new_entropy() { + local entropy_flag="${1}" + + if [ "${entropy_flag}" = "-d" ]; then + get_dilbert_entropy + else + get_arc4cat_entropy + fi +} + +collect_entropy() { + local entropy_flag \ + i entropy_file entropy_subdir + + entropy_flag="${1}" + i=0 + entropy_subdir="${ENTROPY_DIR}/${ENTROPY_SUBDIR_PREFIX}$(date +%s)" + + mkdir -p "${entropy_subdir}" + + ENTROPY_FILE_POSTFIX="" + + echo "Collecting ${MAX_ENTROPY_FILES} entropy files of size ${ENTROPY_FILE_SIZE} ..." + + # We don't call get_shiny_new_entropy() inside the loop as it + # would result in bits of entropy getting dropped on the floor + # between files. While we don't care about the "waste", we do + # care about not being able to test those bits later on. + get_shiny_new_entropy "${entropy_flag}" | while [ "${i}" -lt "${MAX_ENTROPY_FILES}" ]; do + entropy_file="$(printf "${entropy_subdir}/%.6i" "${i}")" + #echo "Creating ${entropy_file}" + dd bs="${ENTROPY_FILE_SIZE}" count=1 of="${entropy_file}" 2>/dev/null + i=$((i + 1)) + done +} + +create_entropy_cache() { + local \ + entropy_file + + entropy_file="${ENTROPY_DIR}/${LARGE_ENTROPY_FILE_NAME}" + + echo "Building a single entropy file '${ENTROPY_CACHE_FILE}' based on the files collected previously ..." 1>&2 + cat_collected_entropy > "${ENTROPY_CACHE_FILE}" +} + +replay_entropy() { + local fast_flag \ + entropy_file + + fast_flag="${1}" + + if [ "${fast_flag}" = "-f" ]; then + entropy_file="${ENTROPY_DIR}/${LARGE_ENTROPY_FILE_NAME}" + + if [ ! -f "${ENTROPY_CACHE_FILE}" ]; then + create_entropy_cache + fi + cat_entropy_cache + else + cat_collected_entropy + fi +} + +warn_about_entropy_reuse() { + echo "$0: Oh noes, we're out of collected entropy. Going back to the beginning." 1>&2 + echo "This shouldn't be a problem as long as no single test sees repeated data." 1>&2 +} + +cat_entropy_cache() { + while true; do + cat "${ENTROPY_CACHE_FILE}" + warn_about_entropy_reuse + done +} + +cat_collected_entropy() { + local \ + i f entropy_collections entropy_subdir entropy_file + + # XXX: Too fucking slow + #entropy_collections="$(find "${ENTROPY_DIR}/" -name "${ENTROPY_SUBDIR_PREFIX}*" -depth 1 -type 1)" + + # Not best practice but at least the performance doesn't suck + # and it works as expected. + + entropy_collections="${ENTROPY_DIR}/${ENTROPY_SUBDIR_PREFIX}"* + i=0 + while true; do + f="$(printf "%.6i" "${i}")" + for entropy_subdir in $entropy_collections; do + entropy_file="${entropy_subdir}/${f}" + #echo "Catting ${entropy_file}" + cat "${entropy_file}" || return 1 + done + i=$((i + 1)) + if [ "${i}" -eq "${MAX_ENTROPY_FILES}" ]; then + warn_about_entropy_reuse + i=0 + fi + done +} + +get_dieharder_tests() { + dieharder -l | awk '/-d/ {print $2}' +} + +analyze_collected_entropy() { + local fast_flag \ + test_number + + fast_flag="${1}" + + # We call replay_entropy() inside the loop to make sure + # the beginning of the collected entropy is checked by all tests + # (instead of having each test start at different offsets). + for test_number in $(get_dieharder_tests); do + replay_entropy ${fast_flag} | dieharder -g 200 -d "${test_number}" + done +} + +get_arc4cat_code() { + cat< +#include +#include + +int main(void) { + char buf[4096]; + + while (1) { + arc4random_buf(buf, sizeof(buf)); + write(1, buf, sizeof(buf)); + } +} +EOF +} + +build_arc4cat() { + mkdir -p "${ARC4CAT_DIR}" + cd "${ARC4CAT_DIR}" + get_arc4cat_code > arc4cat.c + make arc4cat + rm arc4cat.c +} + +usage() { + echo "Looks like you are doing it wrong. Try one of these:" + echo + echo "$0 analyze" + echo "$0 build" + echo "$0 cat" + echo "$0 collect" + return 1 +} + +main() { + local mode="${1}" + + shift + set -e + prepare + + case "${mode}" in + analyze) + analyze_collected_entropy "${@}" + ;; + build) + build_arc4cat + ;; + cat) + cat_collected_entropy + ;; + collect) + collect_entropy "${@}" + ;; + remix) + create_entropy_cache + ;; + *) + usage + ;; + esac +} + +main "${@}" -- 2.11.0 From b76e9c9a98f8781048f28ef07a03c4eb3a2c4652 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 23 Jul 2015 19:16:44 +0200 Subject: [PATCH 134/255] dtrace_consume(): Warn if cpu cores aren't completely in sync ... instead of aborting. While the results may be not completely accurate, in some cases it may not matter. Previously the assertion would sometimes be triggered on a systems with poor timecounters (TSC-low(-100) ACPI-fast(900) i8254(0) dummy(-1000000)). Obtained from: ElectroBSD --- cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c index 04c022b7ad00..39530cf41e83 100644 --- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c +++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -3325,7 +3326,13 @@ dtrace_consume(dtrace_hdl_t *dtp, FILE *fp, if (buf != NULL) { if (first_timestamp == 0) first_timestamp = buf->dtbd_timestamp; - assert(buf->dtbd_timestamp >= first_timestamp); + if (buf->dtbd_timestamp < first_timestamp) { + warnx("cpu clocks out of sync " + "(%ju < %ju; offset: %ju). " + "Results may be incorrect!", + buf->dtbd_timestamp, first_timestamp, + first_timestamp - buf->dtbd_timestamp); + } dt_pq_insert(dtp->dt_bufq, buf); drops[i] = buf->dtbd_drops; -- 2.11.0 From eb6fd8deb4e5233ad4375c0575a4be7209b82d52 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 19:06:37 +0200 Subject: [PATCH 135/255] release: Add ElectroBSD install instructions ... that aren't particularly verbose, though. Obtained from: ElectroBSD --- release/rc.local | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/release/rc.local b/release/rc.local index cb442273c303..0d2fad20c1e1 100755 --- a/release/rc.local +++ b/release/rc.local @@ -37,7 +37,7 @@ if [ $? -eq 0 ]; then else # Serial or other console echo - echo "Welcome to FreeBSD!" + echo "Welcome to ElectroBSD!" echo echo "Please choose the appropriate terminal type for your system." echo "Common console types are:" @@ -62,7 +62,19 @@ if [ -f /etc/installerconfig ]; then exit fi -dialog --backtitle "FreeBSD Installer" --title "Welcome" --extra-button --extra-label "Shell" --ok-label "Install" --cancel-label "Live CD" --yesno "Welcome to FreeBSD! Would you like to begin an installation or use the live CD?" 0 0 +local timeout=15 +if dialog --backtitle "ElectroBSD $(uname -m) on a stick in da house" \ + --title "Careful now, your data may be at risk." \ + --timeout ${timeout} \ + --yesno "To install ElectroBSD, execute cloudiatr after creating a cloudiatr.conf. Understood? You have ${timeout} seconds to respond." \ + 0 0; then + echo "Great. Good luck." +else + echo "That's very unfortunate. If you don't have backups you probably should not continue." +fi + +exit 0; + case $? in $DIALOG_OK) # Install -- 2.11.0 From 7bf9d6070527b8c1ed669993c09b2a4eec3ff8ea Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 28 Jul 2015 15:07:52 +0200 Subject: [PATCH 136/255] etc/rc.d/dumpon: Get dumpdev=AUTO working for GEOM_ELI consumers Obtained from: ElectroBSD --- etc/rc.d/dumpon | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/rc.d/dumpon b/etc/rc.d/dumpon index 66276eb50535..ce630522e12b 100755 --- a/etc/rc.d/dumpon +++ b/etc/rc.d/dumpon @@ -41,6 +41,7 @@ dumpon_start() fi while read dev mp type more ; do [ "${type}" = "swap" ] || continue + dev="${dev%%.eli}" [ -c "${dev}" ] || continue dumpon_try "${dev}" 2>/dev/null && return 0 done Date: Tue, 16 Jun 2015 15:07:06 +0200 Subject: [PATCH 137/255] deadlkres(): (Try to) optionally unlock deadlocked processes to work around USB deadlocks This is work in progress and hasn't been properly tested yet. Obtained from: ElectroBSD --- sys/kern/kern_clock.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index 7734e81518af..7ab290ca4863 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -178,6 +178,7 @@ static const char *blessed[] = { }; static int slptime_threshold = 1800; static int blktime_threshold = 900; +static int unlock_deadlocked_processes = 0; static int sleepfreq = 3; static void @@ -236,9 +237,15 @@ deadlkres(void) * turnstile. */ PROC_UNLOCK(p); - sx_sunlock(&allproc_lock); + if (unlock_deadlocked_processes == 0) { + sx_sunlock(&allproc_lock); panic("%s: possible deadlock detected for %p, blocked for %d ticks\n", - __func__, td, tticks); + __func__, td, tticks); + } else { + printf("%s: possible deadlock detected for %p, blocked for %d ticks. " + "Unlocking process to see what happens. Good luck.\n", __func__, td, tticks); + TD_CLR_LOCK(td); + } } } else if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) { @@ -282,9 +289,15 @@ deadlkres(void) continue; } PROC_UNLOCK(p); - sx_sunlock(&allproc_lock); + if (unlock_deadlocked_processes == 0) { + sx_sunlock(&allproc_lock); panic("%s: possible deadlock detected for %p, blocked for %d ticks\n", - __func__, td, tticks); + __func__, td, tticks); + } else { + printf("%s: possible deadlock detected for %p, blocked for %d ticks. " + "Unlocking process to see what happens. Good luck.\n", __func__, td, tticks); + TD_CLR_LOCK(td); + } } } else thread_unlock(td); @@ -316,6 +329,11 @@ SYSCTL_INT(_debug_deadlkres, OID_AUTO, blktime_threshold, CTLFLAG_RW, "Number of seconds within is valid to block on a turnstile"); SYSCTL_INT(_debug_deadlkres, OID_AUTO, sleepfreq, CTLFLAG_RW, &sleepfreq, 0, "Number of seconds between any deadlock resolver thread run"); +SYSCTL_INT(_debug_deadlkres, OID_AUTO, + unlock_deadlocked_processes_and_see_what_happens, + CTLFLAG_RW, &unlock_deadlocked_processes, 0, + "'Resolve' deadlocks by merely unlocking the locked process. " + "May cause permanent data corruption."); #endif /* DEADLKRES */ void -- 2.11.0 From bd3d121b9d9d8aeab8c963f2b0de0f30eb210136 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 1 Aug 2015 11:42:55 +0200 Subject: [PATCH 138/255] sys/dev/vt: Default to showing splash CPU logo(s) on boot Obtained from: ElectroBSD --- sys/dev/vt/vt_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index e09e8a786b3c..9f54b23d3097 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -137,7 +137,7 @@ static VT_SYSCTL_INT(kbd_panic, 0, "Enable request to panic. " /* Used internally, not a tunable. */ int vt_draw_logo_cpus; -VT_SYSCTL_INT(splash_cpu, 0, "Show logo CPUs during boot"); +VT_SYSCTL_INT(splash_cpu, 1, "Show logo CPUs during boot"); VT_SYSCTL_INT(splash_ncpu, 0, "Override number of logos displayed " "(0 = do not override)"); VT_SYSCTL_INT(splash_cpu_style, 2, "Draw logo style " -- 2.11.0 From 0a8688744ff708afa0e08385f66b4c934c58e0b3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 1 Aug 2015 16:59:55 +0200 Subject: [PATCH 139/255] sys/dev/vt: Default to showing Beastie instead of the orb/sextoy Obtained from: ElectroBSD --- sys/dev/vt/vt_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index 9f54b23d3097..4f7c5ee7c554 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -140,7 +140,7 @@ int vt_draw_logo_cpus; VT_SYSCTL_INT(splash_cpu, 1, "Show logo CPUs during boot"); VT_SYSCTL_INT(splash_ncpu, 0, "Override number of logos displayed " "(0 = do not override)"); -VT_SYSCTL_INT(splash_cpu_style, 2, "Draw logo style " +VT_SYSCTL_INT(splash_cpu_style, 1, "Draw logo style " "(0 = Alternate beastie, 1 = Beastie, 2 = Orb)"); VT_SYSCTL_INT(splash_cpu_duration, 10, "Hide logos after (seconds)"); -- 2.11.0 From 5ca81dc4b0a3929c30a1f6c9e18f803789853b1d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 2 Aug 2015 17:56:01 +0200 Subject: [PATCH 140/255] sys/modules: Detach a couple of ATM-related modules Obtained from: ElectroBSD --- sys/modules/Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 0011024c1d5f..0795a902a43e 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -118,7 +118,6 @@ SUBDIR= \ ${_ex} \ ${_exca} \ ext2fs \ - ${_fatm} \ fdc \ fdescfs \ ${_fe} \ @@ -131,8 +130,7 @@ SUBDIR= \ geom \ ${_glxiic} \ ${_glxsb} \ - gpio \ - hatm \ + ${_gpio} \ hifn \ hme \ ${_hpt27xx} \ @@ -288,7 +286,6 @@ SUBDIR= \ ow \ ${_padlock} \ ${_padlock_rng} \ - patm \ ${_pccard} \ ${_pcfclock} \ pcn \ @@ -496,7 +493,6 @@ _pfsync= pfsync .if ${MK_SOURCELESS_UCODE} != "no" _bce= bce -_fatm= fatm _fxp= fxp _ispfw= ispfw _mwlfw= mwlfw -- 2.11.0 From 91b5527f38ffda9b3fc3ee401eb6d9d7f3e8b6d8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 2 Aug 2015 19:41:56 +0200 Subject: [PATCH 141/255] crypto/openssl: Update OPENSSL_VERSION_TEXT and break 'FIPS mode' Obtained from: ElectroBSD --- crypto/openssl/crypto/opensslv.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/openssl/crypto/opensslv.h b/crypto/openssl/crypto/opensslv.h index 7f52eae087c3..3b8d1108ab1b 100644 --- a/crypto/openssl/crypto/opensslv.h +++ b/crypto/openssl/crypto/opensslv.h @@ -32,9 +32,9 @@ extern "C" { */ # define OPENSSL_VERSION_NUMBER 0x100020afL # ifdef OPENSSL_FIPS -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2j-fips 26 Sep 2016" +# error "Due to being silly, FIPS validation is not supported by ElectroBSD" # else -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2j-freebsd 26 Sep 2016" +# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2j-ElectroBSD 2016-09-26" # endif # define OPENSSL_VERSION_PTEXT " part of " OPENSSL_VERSION_TEXT -- 2.11.0 From 0cc1ee9a1dae3cfcdc2aabef00a18e385fb7d007 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 13:13:09 +0200 Subject: [PATCH 142/255] libstand: Show a message instead of the boring twiddle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obviously the message should be sysctl controlled, but for now it's just a hardcoded 'Die Ärzte' reference which may contain traces of electro beer. Obtained from: ElectroBSD --- lib/libstand/twiddle.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/libstand/twiddle.c b/lib/libstand/twiddle.c index 96ebbbec43c0..b8c1117ea217 100644 --- a/lib/libstand/twiddle.c +++ b/lib/libstand/twiddle.c @@ -48,6 +48,7 @@ void twiddle(u_int callerdiv) { static u_int callercnt, globalcnt, pos; + static const char message[] = "5,6,7,8 - Bullenstaat!"; callercnt++; if (callerdiv > 1 && (callercnt % callerdiv) != 0) @@ -57,8 +58,9 @@ twiddle(u_int callerdiv) if (globaldiv > 1 && (globalcnt % globaldiv) != 0) return; - putchar("|/-\\"[pos++ & 3]); - putchar('\b'); + putchar((pos < sizeof(message)) ? message[pos] : '\b'); + pos++; + pos = pos % sizeof(message); } void -- 2.11.0 From 81892bc9364d8fd1cd6d3d707173164b00d66435 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 4 Aug 2015 12:35:43 +0200 Subject: [PATCH 143/255] libstand: Change twiddle message to Elektrobier Obtained from: ElectroBSD --- lib/libstand/twiddle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libstand/twiddle.c b/lib/libstand/twiddle.c index b8c1117ea217..f23449778cef 100644 --- a/lib/libstand/twiddle.c +++ b/lib/libstand/twiddle.c @@ -48,7 +48,7 @@ void twiddle(u_int callerdiv) { static u_int callercnt, globalcnt, pos; - static const char message[] = "5,6,7,8 - Bullenstaat!"; + static const char message[] = "Elektrobier!\n"; callercnt++; if (callerdiv > 1 && (callercnt % callerdiv) != 0) -- 2.11.0 From ceed745e8457715dd144eab84abd726a0771da9f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 15 Aug 2015 11:29:39 +0200 Subject: [PATCH 144/255] ttys: Reduce default number of active terminals and mark the console as insecure Obtained from: ElectroBSD --- etc/etc.amd64/ttys | 16 ++++++++-------- etc/etc.i386/ttys | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/etc/etc.amd64/ttys b/etc/etc.amd64/ttys index 15eb30dc91ec..457b3d105b10 100644 --- a/etc/etc.amd64/ttys +++ b/etc/etc.amd64/ttys @@ -27,17 +27,17 @@ # # If console is marked "insecure", then init will ask for the root password # when going to single-user mode. -console none unknown off secure +console none unknown off insecure # ttyv0 "/usr/libexec/getty Pc" xterm on secure # Virtual terminals -ttyv1 "/usr/libexec/getty Pc" xterm on secure -ttyv2 "/usr/libexec/getty Pc" xterm on secure -ttyv3 "/usr/libexec/getty Pc" xterm on secure -ttyv4 "/usr/libexec/getty Pc" xterm on secure -ttyv5 "/usr/libexec/getty Pc" xterm on secure -ttyv6 "/usr/libexec/getty Pc" xterm on secure -ttyv7 "/usr/libexec/getty Pc" xterm on secure +ttyv1 "/usr/libexec/getty Pc" xterm off secure +ttyv2 "/usr/libexec/getty Pc" xterm off secure +ttyv3 "/usr/libexec/getty Pc" xterm off secure +ttyv4 "/usr/libexec/getty Pc" xterm off secure +ttyv5 "/usr/libexec/getty Pc" xterm off secure +ttyv6 "/usr/libexec/getty Pc" xterm off secure +ttyv7 "/usr/libexec/getty Pc" xterm off secure ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure # Serial terminals # The 'dialup' keyword identifies dialin lines to login, fingerd etc. diff --git a/etc/etc.i386/ttys b/etc/etc.i386/ttys index 15eb30dc91ec..457b3d105b10 100644 --- a/etc/etc.i386/ttys +++ b/etc/etc.i386/ttys @@ -27,17 +27,17 @@ # # If console is marked "insecure", then init will ask for the root password # when going to single-user mode. -console none unknown off secure +console none unknown off insecure # ttyv0 "/usr/libexec/getty Pc" xterm on secure # Virtual terminals -ttyv1 "/usr/libexec/getty Pc" xterm on secure -ttyv2 "/usr/libexec/getty Pc" xterm on secure -ttyv3 "/usr/libexec/getty Pc" xterm on secure -ttyv4 "/usr/libexec/getty Pc" xterm on secure -ttyv5 "/usr/libexec/getty Pc" xterm on secure -ttyv6 "/usr/libexec/getty Pc" xterm on secure -ttyv7 "/usr/libexec/getty Pc" xterm on secure +ttyv1 "/usr/libexec/getty Pc" xterm off secure +ttyv2 "/usr/libexec/getty Pc" xterm off secure +ttyv3 "/usr/libexec/getty Pc" xterm off secure +ttyv4 "/usr/libexec/getty Pc" xterm off secure +ttyv5 "/usr/libexec/getty Pc" xterm off secure +ttyv6 "/usr/libexec/getty Pc" xterm off secure +ttyv7 "/usr/libexec/getty Pc" xterm off secure ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure # Serial terminals # The 'dialup' keyword identifies dialin lines to login, fingerd etc. -- 2.11.0 From a5b3663abc23c4b311dff4381eaeb7123628ca38 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 21:28:15 +0200 Subject: [PATCH 145/255] usr.sbin: Don't build uathload if MK_SOURCELESS_UCODE is defined It breaks the build if the source tree does not contain the firmware. Obtained from: ElectroBSD --- usr.sbin/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index 67499a10b5c6..f0219a6fa445 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -200,7 +200,12 @@ SUBDIR.${MK_TIMED}+= timed SUBDIR.${MK_TOOLCHAIN}+= config SUBDIR.${MK_TOOLCHAIN}+= crunch SUBDIR.${MK_UNBOUND}+= unbound +# MK_SOURCELESS_UCODE is supposed to affect kernel +# modules only but there is no matching define for +# userland stuff. +.if ${MK_SOURCELESS_UCODE} != "no" SUBDIR.${MK_USB}+= uathload +.endif SUBDIR.${MK_USB}+= uhsoctl SUBDIR.${MK_USB}+= usbconfig SUBDIR.${MK_USB}+= usbdump -- 2.11.0 From 51ae2ea22a80b8915309c05b84d41be8855ef4c6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 21:33:19 +0200 Subject: [PATCH 146/255] Detach fwcontrol from the build It's pointless without firewire(4). Obtained from: ElectroBSD --- usr.sbin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index f0219a6fa445..3d146d6e7704 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -30,7 +30,6 @@ SUBDIR= adduser \ extattrctl \ fifolog \ fstyp \ - fwcontrol \ getfmac \ getpmac \ gstat \ -- 2.11.0 From d9f6d9ca4ac4bc90f7838c1c396b1ee87896a5cd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 23:58:46 +0200 Subject: [PATCH 147/255] sys/x86: Remove another FreeBSD reference from log messages Obtained from: ElectroBSD --- sys/x86/x86/mp_x86.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index 9de533922e95..1d2bed8ac170 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -617,7 +617,7 @@ cpu_mp_announce(void) int cores_per_pkg; int thrs_per_core; - printf("FreeBSD/SMP: "); + printf("ElectroBSD/SMP: "); if (topo_analyze(&topo_root, 1, &pkg_count, &cores_per_pkg, &thrs_per_core)) { printf("%d package(s)", pkg_count); @@ -631,7 +631,7 @@ cpu_mp_announce(void) printf("\n"); if (disabled_cpus) { - printf("FreeBSD/SMP Online: "); + printf("ElectroBSD/SMP Online: "); if (topo_analyze(&topo_root, 0, &pkg_count, &cores_per_pkg, &thrs_per_core)) { printf("%d package(s)", pkg_count); -- 2.11.0 From e0c33c110b0526e6c2a8861d7552f788ad7f05aa Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 20 Aug 2015 10:12:27 +0200 Subject: [PATCH 148/255] pw(8): Increase minimal random password length to 32 Obtained from: ElectroBSD --- usr.sbin/pw/pw_user.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.sbin/pw/pw_user.c b/usr.sbin/pw/pw_user.c index 75e7fb6e5c88..85db873017bd 100644 --- a/usr.sbin/pw/pw_user.c +++ b/usr.sbin/pw/pw_user.c @@ -512,7 +512,7 @@ pw_password(struct userconf * cnf, char const * user, bool dryrun) switch (cnf->default_password) { case -1: /* Random password */ - l = (arc4random() % 8 + 8); /* 8 - 16 chars */ + l = 32 + (arc4random() % 8); /* 32 - 40 chars */ for (i = 0; i < l; i++) pwbuf[i] = chars[arc4random_uniform(sizeof(chars)-1)]; pwbuf[i] = '\0'; -- 2.11.0 From aa29ee857fc1497b377d2dddf501cf983ddd27ed Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 24 Aug 2015 14:31:05 +0200 Subject: [PATCH 149/255] Makefile.inc1: Respect WITHOUT_TCSH Obtained from: ElectroBSD --- Makefile.inc1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 33904d920c69..41c6a78dbfcc 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1760,8 +1760,13 @@ _gcc_tools= gnu/usr.bin/cc/cc_tools _rescue=rescue/rescue .endif +.if ${MK_TCSH} != "no" +_tcsh=bin/csh +.endif + +build-tools: .MAKE .for _tool in \ - bin/csh \ + ${_tcsh} \ bin/sh \ ${LOCAL_TOOL_DIRS} \ lib/ncurses/ncurses \ @@ -1912,7 +1917,7 @@ native-xtools: .PHONY bin/cat \ bin/chmod \ bin/cp \ - bin/csh \ + ${_tcsh} \ bin/echo \ bin/expr \ bin/hostname \ -- 2.11.0 From 7a5a90ff7994b488bef2e54a91ac8955902729ef Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 4 Jun 2016 11:38:43 +0200 Subject: [PATCH 150/255] Detach a couple of rc scripts that aren't relevant for ElectroBSD XXX: Among other things this detaches bgfsck which could still be considered relevant for the memstick images. Obtained from: ElectroBSD --- etc/rc.d/Makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index ec3ad74a4bae..ac3dda4100a2 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -16,7 +16,6 @@ FILES= DAEMON \ archdep \ auditd \ auditdistd \ - bgfsck \ ${_blacklistd} \ ${_bluetooth} \ bridge \ @@ -32,7 +31,6 @@ FILES= DAEMON \ dhclient \ dmesg \ dumpon \ - fsck \ gbde \ geli \ geli2 \ @@ -45,7 +43,6 @@ FILES= DAEMON \ hostname \ iovctl \ ip6addrctl \ - ipfilter \ ipfs \ ipfw \ ipmon \ @@ -70,7 +67,6 @@ FILES= DAEMON \ mdconfig2 \ mountd \ msgs \ - natd \ netif \ netoptions \ netwait \ @@ -87,8 +83,6 @@ FILES= DAEMON \ pf \ pflog \ pfsync \ - ppp \ - pppoed \ pwcheck \ quota \ random \ @@ -105,7 +99,6 @@ FILES= DAEMON \ securelevel \ serial \ soft-protection \ - sppp \ statd \ static_arp \ static_ndp \ -- 2.11.0 From 55126a4a0c0d33c922dbc0906067645559c7ff2f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 9 Jul 2015 14:55:50 +0200 Subject: [PATCH 151/255] etc/pkg: Add and install ElectroBSD.conf and a pubkey file ... for testing purposes. While at it, detach FreeBSD.conf and the corresponding fingerprint file from the build. Obtained from: ElectroBSD --- etc/pkg/ElectroBSD.conf | 21 +++++++++++++++++++++ etc/pkg/Makefile | 2 +- etc/pkg/electrobsd-test-repo.pub | 14 ++++++++++++++ share/keys/pkg/trusted/Makefile | 2 +- share/keys/pkg/trusted/electrobsd-test-key.conf | 2 ++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 etc/pkg/ElectroBSD.conf create mode 100644 etc/pkg/electrobsd-test-repo.pub create mode 100644 share/keys/pkg/trusted/electrobsd-test-key.conf diff --git a/etc/pkg/ElectroBSD.conf b/etc/pkg/ElectroBSD.conf new file mode 100644 index 000000000000..4bf82d992bee --- /dev/null +++ b/etc/pkg/ElectroBSD.conf @@ -0,0 +1,21 @@ +ElectroBSD: { + # Note that accessing this repository requires a local proxy + # that port forwards the incomming http requests to an authorized + # Tor client that forwards them to: http://gkpssb7hd77qznoa.onion/ + # + # To get the required secret you have to ask someone who knows it and + # come up with a good reason why you want to run unreproducible packages + # instead of building your packages from source. + # + # Example bootstrapping setup: + # Your server: pkg -> sshd + # | + # Your client: --> ssh -> privoxy -> tor + # | + # Outside your control: --> Tor network -> Tor HS + url: "http://127.0.0.1:8000/packages/${ABI}/", + mirror_type: "NONE", + signature_type: "pubkey", + pubkey: "/etc/pkg/electrobsd-test-repo.pub", + enabled: yes +} diff --git a/etc/pkg/Makefile b/etc/pkg/Makefile index abc8c64ca5cb..614989f36674 100644 --- a/etc/pkg/Makefile +++ b/etc/pkg/Makefile @@ -2,7 +2,7 @@ NO_OBJ= -FILES= FreeBSD.conf +FILES= ElectroBSD.conf electrobsd-test-repo.pub FILESDIR= /etc/pkg FILESMODE= 644 diff --git a/etc/pkg/electrobsd-test-repo.pub b/etc/pkg/electrobsd-test-repo.pub new file mode 100644 index 000000000000..ee213cb062b1 --- /dev/null +++ b/etc/pkg/electrobsd-test-repo.pub @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtwIgcSgQT2QJvxwDiKhh +xYidaxWaJL/EHXnYH8TpiujNeg8yZUCiT07MChw/wO3KrVY3SlhFkl8MHnwaFWNS +jSJFNQIHoVo8of0juDXhIMMY0DTVFVpSfgq+Ea0SrVZawQchcc7XODjp7mQOH0Nz +Z0FUpQ4z7TsdrPV/Kj/SQteK7Q0qAnDGT5GBxb+4m5qmNbv74GVpWmxJ5kC4OY3g +v8oTRZ5wG/mf5nKG+QXGKbcfoK+fd+NL9gp5awhvnp5cRo2+ZrQvnYKy5ym9O+jM +9hv5bL7OYIW7gErPdQd0+SFejcDg4TamY824krkjgIf7pnRVAuuSHiy8PaHmmNMd +StFo8IW6687C0X4gNuaPtR1TCkVZugrgev/w5GPp765HmE4WvrtPU4u9/EvWO2L6 +2bHmsJqNIGyq1lfHk/cx49CEhDPjRcYin8MKKzRFUR7CWiNDBz2GK6Lcfd78cG5I +HGJAthhi3rOOnLsTMuIQ/+GkVCDNwweLxSoLQUzGnheHqBjYyZSfZ8u9I/OKtwp3 +OMhGOD1/yuRaat+QER5VwiDP3AxcIlq6aNNytJW6l/kZZLVsr99YmROyYsTps0Zm +OXqxQFsZ6zzlYO95HP4ITJ7ubk84YPTvxbQ6LqsijqRI322aNvxxeEWyFxXpyCpp +uq50HtVoVX1a4ONsh/9l75sCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/share/keys/pkg/trusted/Makefile b/share/keys/pkg/trusted/Makefile index e55977c0a9a9..f55afc979969 100644 --- a/share/keys/pkg/trusted/Makefile +++ b/share/keys/pkg/trusted/Makefile @@ -1,6 +1,6 @@ # $FreeBSD$ -FILES= pkg.freebsd.org.2013102301 +FILES= electrobsd-test-key.conf FILESDIR= ${SHAREDIR}/keys/pkg/trusted FILESMODE= 644 diff --git a/share/keys/pkg/trusted/electrobsd-test-key.conf b/share/keys/pkg/trusted/electrobsd-test-key.conf new file mode 100644 index 000000000000..e81a919e0ab2 --- /dev/null +++ b/share/keys/pkg/trusted/electrobsd-test-key.conf @@ -0,0 +1,2 @@ +function: "sha256" +fingerprint: "8b1ef90f16ccc99342fb204f7ff57cdd31449e5ce1baa10c0b1b09593c44219d" -- 2.11.0 From 4fb452fba600f517d9dddf0ea18b060cc34081d1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 5 Sep 2015 22:46:09 +0200 Subject: [PATCH 152/255] dtrace: Add look-who-is-reaping It can be used to monitor and finetune the ARC cache reaper. The stack trace obviously isn't particular useful anymore now that the reaper is called from a single location. Obtained from: ElectroBSD --- share/dtrace/look-who-is-reaping | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 share/dtrace/look-who-is-reaping diff --git a/share/dtrace/look-who-is-reaping b/share/dtrace/look-who-is-reaping new file mode 100755 index 000000000000..0db6324a64f7 --- /dev/null +++ b/share/dtrace/look-who-is-reaping @@ -0,0 +1,58 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * look-who-is-reaping + * + * Collect stack traces for reap_arc_caches(). + * + * Copyright (c) 2015 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + printf("%Y: Look who's (ARC cache) reaping .... Hit CTRL-C to exit.\n", walltimestamp); +} + +fbt::reap_arc_caches:entry +{ + @reapers[probefunc,stack(3)] = count(); + @reapers_total[probefunc,stack(3)] = count(); + reaped++ +} + +tick-10s +/reaped == 0/ +{ + printf("%Y: The ARC cache reaper has not been seen for 10 seconds!\n", walltimestamp); +} + +tick-10s +/reaped/ +{ + printf("%Y: The ARC reaper was called:\n", walltimestamp); + printa(@reapers); + trunc(@reapers); + reaped = 0 +} + +END +/reaped/ +{ + printf("%Y: The ARC reaper stats total:\n", walltimestamp); + printa(@reapers_total); +} -- 2.11.0 From 54e9a4bc768ae6fe2ec42fd501f7585a57ac140b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Sep 2015 20:07:22 +0200 Subject: [PATCH 153/255] dtrace: Import fbt-time which llquantizes the time spent in a given kernel function Among other things this can be useful for general system tuning and analysis of lock contention (once you know where to expect it). Obtained from: ElectroBSD --- share/dtrace/fbt-time | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 share/dtrace/fbt-time diff --git a/share/dtrace/fbt-time b/share/dtrace/fbt-time new file mode 100755 index 000000000000..27a950b63a0c --- /dev/null +++ b/share/dtrace/fbt-time @@ -0,0 +1,80 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * fbt-time + * + * Measures the time spent in a given kernel function and llquantizes + * the results for the traced calls. Example output can be seen at: + * http://www.fabiankeil.de/gehacktes/electrobsd/zfs-arc-tuning/ + * + * WARNING: Tracing kernel functions that get executed several thousand + * times per second may negatively impact system performance + * and can even cause brief lockups. Before using this script + * on production systems you may want to sample the traced + * kernel function first. + * + * XXX: This script should be fleshed out to be more resilient to + * incorrect input and provide more awesome stats. A better name + * would be nice, too. + * + * Copyright (c) 2014 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + traced_function = $$1; + + multiple = "microseconds"; + divisor = 1000; + executions = 0; + + this->timestamp = walltimestamp; + this->msecs = (this->timestamp / 1000000) % 1000; + printf("%Y.%.3d: Tracing '%s'. Hit CTRL-C to exit.\n", + this->timestamp, this->msecs, traced_function); +} + +fbt::$$1:entry +{ + self->timestamp = timestamp; + executions++; +} + +fbt::$$1:return +/self->timestamp/ +{ + this->elapsed = (timestamp - self->timestamp) / divisor; + @elapsed_time = llquantize(this->elapsed, 10, 0, 5, 10); + @elapsed_avg["Average"] = avg(this->elapsed); + self->elapsed = 0; +} + +tick-10sec, +END +/executions/ +{ + printf("%Y: Runtime for %s() in %s\n", walltimestamp, traced_function, multiple); + printa(@elapsed_avg); + printa(@elapsed_time); +} + +END +/executions == 0/ +{ + printf("%Y: Looks like %s() wasn't called yet.\n", walltimestamp, traced_function); +} -- 2.11.0 From 5c3048f06992c31e18ef4df592c03c1e55a0ff7c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Mar 2015 13:06:05 +0100 Subject: [PATCH 154/255] ifconfig: Implement MAC address randomization Usage: ifconfig $nic ether random Inspired by the OpenBSD code which unfortunately can't be imported directly due to code differences. Obtained from: ElectroBSD --- sbin/ifconfig/af_link.c | 8 ++++++++ sbin/ifconfig/ifconfig.8 | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sbin/ifconfig/af_link.c b/sbin/ifconfig/af_link.c index c96cbd2ce92e..c32319b30398 100644 --- a/sbin/ifconfig/af_link.c +++ b/sbin/ifconfig/af_link.c @@ -90,6 +90,14 @@ link_getaddr(const char *addr, int which) if (which != ADDR) errx(1, "can't set link-level netmask or broadcast"); + if (!strcmp(addr, "random")) { + sa->sa_family = AF_LINK; + sa->sa_len = ETHER_ADDR_LEN; + arc4random_buf(&sa->sa_data, sa->sa_len); + /* Make sure it's a non-multicast hardware address */ + sa->sa_data[0] &= 0xfc; + return; + } if ((temp = malloc(strlen(addr) + 2)) == NULL) errx(1, "malloc failed"); temp[0] = ':'; diff --git a/sbin/ifconfig/ifconfig.8 b/sbin/ifconfig/ifconfig.8 index e8db561f45a0..05d23d358b01 100644 --- a/sbin/ifconfig/ifconfig.8 +++ b/sbin/ifconfig/ifconfig.8 @@ -141,7 +141,10 @@ parameter below for more information. The link-level .Pq Dq link address -is specified as a series of colon-separated hex digits. +is specified as a series of colon-separated hex digits +or, if the address is +.Dq random , +will be chosen randomly. This can be used to, for example, set a new MAC address on an Ethernet interface, though the mechanism used is not Ethernet specific. -- 2.11.0 From 06c812d61c58a9d1dd24f239eb992d090dd932bd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Mar 2015 14:22:43 +0100 Subject: [PATCH 155/255] Import rc.d/rether which randomizes MAC addresses ... provided ifconfig supports "either random". Obtained from: ElectroBSD --- etc/rc.d/Makefile | 1 + etc/rc.d/rether | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100755 etc/rc.d/rether diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index ac3dda4100a2..578e69ababd5 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -89,6 +89,7 @@ FILES= DAEMON \ rarpd \ rctl \ resolv \ + rether \ root \ route6d \ routing \ diff --git a/etc/rc.d/rether b/etc/rc.d/rether new file mode 100755 index 000000000000..51e860353458 --- /dev/null +++ b/etc/rc.d/rether @@ -0,0 +1,97 @@ +#!/bin/sh +# +########################################################################### +# +# rether - Randomizes MAC addresses +# +# Add the following line to /etc/rc.conf to randomize the MAC +# address for all recognized network interfaces that got one +# at startup: +# +# rether_enable="YES" +# +# You can specify the interfaces manually like this: +# +# rether_interfaces="bge0 iwn0" +# +# By default rether runs before netif so cloned devices aren't +# expected to exist yet. In case of wlan clones the MAC address +# of the parent is inherited so as long as it's randomized the +# clone should be fine too. +# +# Note that MAC address mismatches between clone and parent may +# prevent the clone from working as expected. If you intend to +# run this script after the system is up, you may want to +# explicitly set rether_interfaces to skip clones. +# +# Rether requires an ifconfig version that understands "ether random". +# +########################################################################### +# +# Copyright (c) 2014 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# PROVIDE: rether +# REQUIRE: FILESYSTEMS +# BEFORE: netif + +. /etc/rc.subr + +name="rether" +rcvar="rether_enable" + +rether_enable="${rether_enable-NO}" + +start_cmd="rether_start" +stop_cmd=":" + +# For the reason given above, it may make sense to skip cloned devices. +get_ethernet_interfaces() { + ifconfig -l ether +} + +get_ethernet_address() { + local interface="${1}" + + ifconfig "${interface}" ether | awk '/ether/ {print $2}' +} + +rether_start() { + local \ + interface \ + ethernet_address + + if [ -z "${rether_interfaces}" ]; then + rether_interfaces="$(get_ethernet_interfaces)" + fi + + echo "Randomizing MAC addresses for: ${rether_interfaces}" + + for interface in $rether_interfaces; + do + ethernet_address="$(get_ethernet_address "${interface}")" + + ifconfig "${interface}" ether random + + if [ "${ethernet_address}" = "$(get_ethernet_address "${interface}")" ]; then + echo "Failed to randomize MAC address for ${interface}: ${ethernet_address}" + fi + done +} + +load_rc_config "${name}" +run_rc_command "${1}" -- 2.11.0 From 69fa9dfd215fb24c94dab04782a154b65a639008 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Sep 2015 17:20:31 +0200 Subject: [PATCH 156/255] share/mk/src.opts.mk: Detach LIB32 from the build Obtained from: ElectroBSD --- share/mk/src.opts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index d71083d52e77..d4bd454b462f 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -105,7 +105,6 @@ __DEFAULT_YES_OPTIONS = \ LDNS \ LDNS_UTILS \ LEGACY_CONSOLE \ - LIB32 \ LIBPTHREAD \ LIBTHR \ LOCALES \ @@ -182,6 +181,7 @@ __DEFAULT_NO_OPTIONS += \ GCOV \ IPFILTER \ ISCSI \ + LIB32 \ NDIS \ RBOOTD \ PC_SYSINSTALL \ -- 2.11.0 From 212f2cc2447345dd2df2a027eb246653706a6933 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 15:19:51 +0200 Subject: [PATCH 157/255] Prevent inlining of vm_pageout_scan() to allow fbt tracing Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index ed9d439af87f..a3f8774c9160 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -849,7 +849,7 @@ unlock_mp: * Returns true if pass was zero or enough pages were freed by the inactive * queue scan to meet the target. */ -static bool +static bool __noinline vm_pageout_scan(struct vm_domain *vmd, int pass) { vm_page_t m, next; -- 2.11.0 From 54dd3833e32d3e4c07ef1a6bd5c02710ec071af4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 15:33:12 +0200 Subject: [PATCH 158/255] Attempt to fix MAC address randomization after r287197 Obtained from: ElectroBSD --- etc/Makefile | 1 + etc/{rc.d/rether => mac-randomization.subr} | 35 ++++++----------------------- etc/rc.d/Makefile | 1 - etc/rc.d/netif | 3 +++ 4 files changed, 11 insertions(+), 29 deletions(-) rename etc/{rc.d/rether => mac-randomization.subr} (74%) diff --git a/etc/Makefile b/etc/Makefile index fe518d283d19..67bb39e701d9 100644 --- a/etc/Makefile +++ b/etc/Makefile @@ -31,6 +31,7 @@ BIN1= crontab \ login.access \ login.conf \ mac.conf \ + mac-randomization.subr \ motd \ netconfig \ network.subr \ diff --git a/etc/rc.d/rether b/etc/mac-randomization.subr similarity index 74% rename from etc/rc.d/rether rename to etc/mac-randomization.subr index 51e860353458..d486bf454591 100755 --- a/etc/rc.d/rether +++ b/etc/mac-randomization.subr @@ -2,7 +2,7 @@ # ########################################################################### # -# rether - Randomizes MAC addresses +# Sub routines to randomizes MAC addresses after r287197 (WIP!) # # Add the following line to /etc/rc.conf to randomize the MAC # address for all recognized network interfaces that got one @@ -14,16 +14,6 @@ # # rether_interfaces="bge0 iwn0" # -# By default rether runs before netif so cloned devices aren't -# expected to exist yet. In case of wlan clones the MAC address -# of the parent is inherited so as long as it's randomized the -# clone should be fine too. -# -# Note that MAC address mismatches between clone and parent may -# prevent the clone from working as expected. If you intend to -# run this script after the system is up, you may want to -# explicitly set rether_interfaces to skip clones. -# # Rether requires an ifconfig version that understands "ether random". # ########################################################################### @@ -45,21 +35,8 @@ # ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. ########################################################################### -# PROVIDE: rether -# REQUIRE: FILESYSTEMS -# BEFORE: netif - -. /etc/rc.subr - -name="rether" -rcvar="rether_enable" - rether_enable="${rether_enable-NO}" -start_cmd="rether_start" -stop_cmd=":" - -# For the reason given above, it may make sense to skip cloned devices. get_ethernet_interfaces() { ifconfig -l ether } @@ -70,11 +47,16 @@ get_ethernet_address() { ifconfig "${interface}" ether | awk '/ether/ {print $2}' } -rether_start() { +randomize_mac_addresses() { local \ interface \ ethernet_address + if [ "${rether_enable}" = "NO" ]; then + echo "Not randomizing any MAC addresses!" + return 0 + fi + if [ -z "${rether_interfaces}" ]; then rether_interfaces="$(get_ethernet_interfaces)" fi @@ -92,6 +74,3 @@ rether_start() { fi done } - -load_rc_config "${name}" -run_rc_command "${1}" diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 578e69ababd5..ac3dda4100a2 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -89,7 +89,6 @@ FILES= DAEMON \ rarpd \ rctl \ resolv \ - rether \ root \ route6d \ routing \ diff --git a/etc/rc.d/netif b/etc/rc.d/netif index a1543e63e704..6b52c50e7f4f 100755 --- a/etc/rc.d/netif +++ b/etc/rc.d/netif @@ -32,6 +32,7 @@ . /etc/rc.subr . /etc/network.subr +. /etc/mac-randomization.subr name="netif" desc="Network interface setup" @@ -74,6 +75,8 @@ netif_start() # Create cloned interfaces clone_up $cmdifn + randomize_mac_addresses + # Rename interfaces. ifnet_rename $cmdifn -- 2.11.0 From 4ef623adc84ad9486d3bbf7699f69c8c63e72890 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 09:53:58 +0200 Subject: [PATCH 159/255] sys/boot/i386: Detach libfirewire from the build Obtained from: ElectroBSD --- sys/boot/i386/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/i386/Makefile b/sys/boot/i386/Makefile index 0c3daed185ee..d42c7b9dc994 100644 --- a/sys/boot/i386/Makefile +++ b/sys/boot/i386/Makefile @@ -3,7 +3,7 @@ .include SUBDIR= mbr pmbr boot0 boot0sio btx boot2 cdboot gptboot \ - libi386 libfirewire loader + libi386 loader # special boot programs, 'self-extracting boot2+loader' SUBDIR+= pxeldr -- 2.11.0 From 0114655e4e6574d016c39174d8d09625b2ae3239 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 29 Sep 2015 22:31:45 +0200 Subject: [PATCH 160/255] README: Spell out GPL correctly, the G does not stand for GNU Obtained from: ElectroBSD --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index b863557e4baa..fe02a3852c3f 100644 --- a/README +++ b/README @@ -45,8 +45,8 @@ crypto Cryptography stuff (see crypto/README). etc Template files for /etc. -gnu Various commands and libraries under the GNU Public License. - Please see gnu/COPYING* for more information. +gnu Various commands and libraries under the GNU General Public + License. Please see gnu/COPYING* for more information. include System include files. -- 2.11.0 From 2165caa60a47e0128b7bd249115d44aca8f6c324 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 30 Sep 2015 21:15:20 +0200 Subject: [PATCH 161/255] Add two DTrace scripts. WIP Obtained from: ElectroBSD --- share/dtrace/io-breakdown | 19 +++++++++++++++++++ share/dtrace/open-ios | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100755 share/dtrace/io-breakdown create mode 100755 share/dtrace/open-ios diff --git a/share/dtrace/io-breakdown b/share/dtrace/io-breakdown new file mode 100755 index 000000000000..ee9e592f1bb5 --- /dev/null +++ b/share/dtrace/io-breakdown @@ -0,0 +1,19 @@ +#!/usr/sbin/dtrace -s + +/* This is the example from dtrace_io(4) */ + +#pragma D option quiet +#pragma D option dynvarsize=16m + +#pragma D option quiet + +io:::start +{ + @[args[1]->device_name, execname, pid] = sum(args[0]->bio_bcount); +} + +END +{ + printf("%10s %20s %10s %15s", "DEVICE", "APP", "PID", "BYTES"); + printa("%10s %20s %10d %15@d", @); +} diff --git a/share/dtrace/open-ios b/share/dtrace/open-ios new file mode 100755 index 000000000000..ee8bbe0d922d --- /dev/null +++ b/share/dtrace/open-ios @@ -0,0 +1,31 @@ +#!/usr/sbin/dtrace -s + +#pragma D option quiet +#pragma D option dynvarsize=16m + +dtrace:::BEGIN +{ + printf("Tracing... Hit Ctrl-C to end.\n"); +} + +io:::start +{ + printf("%Y: Device name\n", walltimestamp); + print(args[1]->device_name); + start_time[arg0] = timestamp; + @open_ios = count(); +} + +io:::done +/this->start = start_time[arg0]/ +{ + @done_ios = count(); +} + +tick-10s +{ + printf("%Y: Started ios", walltimestamp); + printa(@open_ios); + printf("%Y: Done ios", walltimestamp); + printa(@done_ios); +} -- 2.11.0 From 4d6133e0a59e0c54adb479b2cc60ed0e2f40f109 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 13:04:33 +0200 Subject: [PATCH 162/255] games/fortune/datfiles: Do not install non-free gerrold.limerick Obtained from: ElectroBSD --- usr.bin/fortune/datfiles/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.bin/fortune/datfiles/Makefile b/usr.bin/fortune/datfiles/Makefile index 1eabaa4cdc61..5d979d9f21ac 100644 --- a/usr.bin/fortune/datfiles/Makefile +++ b/usr.bin/fortune/datfiles/Makefile @@ -5,7 +5,7 @@ DB= fortunes freebsd-tips murphy startrek zippy # TO AVOID INSTALLING THE POTENTIALLY OFFENSIVE FORTUNES, COMMENT OUT THE # NEXT LINE. -DB+= limerick murphy-o gerrold.limerick +DB+= limerick murphy-o BLDS= ${DB:S/$/.dat/} FILES= ${DB} ${BLDS} -- 2.11.0 From c8c12c0bae671669d5ada9febc1cbc4e0c58744e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 12:53:54 +0200 Subject: [PATCH 163/255] fortune/datafiles: Add German fundamental rights (poorly formatted) Enforcing these rights is left as an excercise for the reader. Obtained from: ElectroBSD --- usr.bin/fortune/datfiles/Makefile | 2 +- usr.bin/fortune/datfiles/grundrechte | 346 +++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 usr.bin/fortune/datfiles/grundrechte diff --git a/usr.bin/fortune/datfiles/Makefile b/usr.bin/fortune/datfiles/Makefile index 5d979d9f21ac..0f25ce005278 100644 --- a/usr.bin/fortune/datfiles/Makefile +++ b/usr.bin/fortune/datfiles/Makefile @@ -1,7 +1,7 @@ # @(#)Makefile 8.2 (Berkeley) 4/19/94 # $FreeBSD$ -DB= fortunes freebsd-tips murphy startrek zippy +DB= grundrechte fortunes freebsd-tips murphy startrek zippy # TO AVOID INSTALLING THE POTENTIALLY OFFENSIVE FORTUNES, COMMENT OUT THE # NEXT LINE. diff --git a/usr.bin/fortune/datfiles/grundrechte b/usr.bin/fortune/datfiles/grundrechte new file mode 100644 index 000000000000..c9a098878b17 --- /dev/null +++ b/usr.bin/fortune/datfiles/grundrechte @@ -0,0 +1,346 @@ +% +Grundgesetz Artikel 1 +(1) Die Würde des Menschen ist unantastbar. Sie zu achten und zu +schützen ist Verpflichtung aller staatlichen Gewalt. +(2) Das Deutsche Volk bekennt sich darum zu unverletzlichen und +unveräußerlichen Menschenrechten als Grundlage jeder menschlichen +Gemeinschaft, des Friedens und der Gerechtigkeit in der Welt. +(3) Die nachfolgenden Grundrechte binden Gesetzgebung, vollziehende +Gewalt und Rechtsprechung als unmittelbar geltendes Recht. + % +Grundgesetz Artikel 2 +(1) Jeder hat das Recht auf die freie Entfaltung seiner +Persönlichkeit, soweit er nicht die Rechte anderer verletzt und nicht +gegen die verfassungsmäßige Ordnung oder das Sittengesetz verstößt. +(2) Jeder hat das Recht auf Leben und körperliche Unversehrtheit. Die +Freiheit der Person ist unverletzlich. In diese Rechte darf nur auf +Grund eines Gesetzes eingegriffen werden. +% +Grundgesetz Artikel 3 +(1) Alle Menschen sind vor dem Gesetz gleich. +(2) Männer und Frauen sind gleichberechtigt. Der Staat fördert die +tatsächliche Durchsetzung der Gleichberechtigung von Frauen und +Männern und wirkt auf die Beseitigung bestehender Nachteile hin. +(3) Niemand darf wegen seines Geschlechtes, seiner Abstammung, seiner +Rasse, seiner Sprache, seiner Heimat und Herkunft, seines Glaubens, +seiner religiösen oder politischen Anschauungen benachteiligt oder +bevorzugt werden. Niemand darf wegen seiner Behinderung benachteiligt +werden. +% +Grundgesetz Artikel 4 +(1) Die Freiheit des Glaubens, des Gewissens und die Freiheit des +religiösen und weltanschaulichen Bekenntnisses sind unverletzlich. +(2) Die ungestörte Religionsausübung wird gewährleistet. +(3) Niemand darf gegen sein Gewissen zum Kriegsdienst mit der Waffe +gezwungen werden. Das Nähere regelt ein Bundesgesetz. +% +Grundgesetz Artikel 5 +(1) Jeder hat das Recht, seine Meinung in Wort, Schrift und Bild frei +zu äußern und zu verbreiten und sich aus allgemein zugänglichen +Quellen ungehindert zu unterrichten. Die Pressefreiheit und die +Freiheit der Berichterstattung durch Rundfunk und Film werden +gewährleistet. Eine Zensur findet nicht statt. +(2) Diese Rechte finden ihre Schranken in den Vorschriften der +allgemeinen Gesetze, den gesetzlichen Bestimmungen zum Schutze der +Jugend und in dem Recht der persönlichen Ehre. +(3) Kunst und Wissenschaft, Forschung und Lehre sind frei. Die +Freiheit der Lehre entbindet nicht von der Treue zur Verfassung. +% +Grundgesetz Artikel 6 +(1) Ehe und Familie stehen unter dem besonderen Schutze der staatlichen Ordnung. +(2) Pflege und Erziehung der Kinder sind das natürliche Recht der +Eltern und die zuvörderst ihnen obliegende Pflicht. Über ihre +Betätigung wacht die staatliche Gemeinschaft. +(3) Gegen den Willen der Erziehungsberechtigten dürfen Kinder nur auf +Grund eines Gesetzes von der Familie getrennt werden, wenn die +Erziehungsberechtigten versagen oder wenn die Kinder aus anderen +Gründen zu verwahrlosen drohen. +(4) Jede Mutter hat Anspruch auf den Schutz und die Fürsorge der Gemeinschaft. +(5) Den unehelichen Kindern sind durch die Gesetzgebung die gleichen +Bedingungen für ihre leibliche und seelische Entwicklung und ihre +Stellung in der Gesellschaft zu schaffen wie den ehelichen Kindern. +% +Grundgesetz Artikel 7 +(1) Das gesamte Schulwesen steht unter der Aufsicht des Staates. +(2) Die Erziehungsberechtigten haben das Recht, über die Teilnahme des +Kindes am Religionsunterricht zu bestimmen. +(3) Der Religionsunterricht ist in den öffentlichen Schulen mit +Ausnahme der bekenntnisfreien Schulen ordentliches +Lehrfach. Unbeschadet des staatlichen Aufsichtsrechtes wird der +Religionsunterricht in Übereinstimmung mit den Grundsätzen der +Religionsgemeinschaften erteilt. Kein Lehrer darf gegen seinen Willen +verpflichtet werden, Religionsunterricht zu erteilen. +(4) Das Recht zur Errichtung von privaten Schulen wird +gewährleistet. Private Schulen als Ersatz für öffentliche Schulen +bedürfen der Genehmigung des Staates und unterstehen den +Landesgesetzen. Die Genehmigung ist zu erteilen, wenn die privaten +Schulen in ihren Lehrzielen und Einrichtungen sowie in der +wissenschaftlichen Ausbildung ihrer Lehrkräfte nicht hinter den +öffentlichen Schulen zurückstehen und eine Sonderung der Schüler nach +den Besitzverhältnissen der Eltern nicht gefördert wird. Die +Genehmigung ist zu versagen, wenn die wirtschaftliche und rechtliche +Stellung der Lehrkräfte nicht genügend gesichert ist. +(5) Eine private Volksschule ist nur zuzulassen, wenn die +Unterrichtsverwaltung ein besonderes pädagogisches Interesse anerkennt +oder, auf Antrag von Erziehungsberechtigten, wenn sie als +Gemeinschaftsschule, als Bekenntnis- oder Weltanschauungsschule +errichtet werden soll und eine öffentliche Volksschule dieser Art in +der Gemeinde nicht besteht. +(6) Vorschulen bleiben aufgehoben. +% +Grundgesetz Artikel 8 +(1) Alle Deutschen haben das Recht, sich ohne Anmeldung oder Erlaubnis +friedlich und ohne Waffen zu versammeln. +(2) Für Versammlungen unter freiem Himmel kann dieses Recht durch +Gesetz oder auf Grund eines Gesetzes beschränkt werden. +% +Grundgesetz Artikel 9 +(1) Alle Deutschen haben das Recht, Vereine und Gesellschaften zu bilden. +(2) Vereinigungen, deren Zwecke oder deren Tätigkeit den Strafgesetzen +zuwiderlaufen oder die sich gegen die verfassungsmäßige Ordnung oder +gegen den Gedanken der Völkerverständigung richten, sind verboten. +(3) Das Recht, zur Wahrung und Förderung der Arbeits- und +Wirtschaftsbedingungen Vereinigungen zu bilden, ist für jedermann und +für alle Berufe gewährleistet. Abreden, die dieses Recht einschränken +oder zu behindern suchen, sind nichtig, hierauf gerichtete Maßnahmen +sind rechtswidrig. Maßnahmen nach den Artikeln 12a, 35 Abs. 2 und 3, +Artikel 87a Abs. 4 und Artikel 91 dürfen sich nicht gegen +Arbeitskämpfe richten, die zur Wahrung und Förderung der Arbeits- und +Wirtschaftsbedingungen von Vereinigungen im Sinne des Satzes 1 geführt +werden. +% +Grundgesetz Artikel 10 +(1) Das Briefgeheimnis sowie das Post- und Fernmeldegeheimnis sind +unverletzlich. +(2) Beschränkungen dürfen nur auf Grund eines Gesetzes angeordnet +werden. Dient die Beschränkung dem Schutze der freiheitlichen +demokratischen Grundordnung oder des Bestandes oder der Sicherung des +Bundes oder eines Landes, so kann das Gesetz bestimmen, daß sie dem +Betroffenen nicht mitgeteilt wird und daß an die Stelle des +Rechtsweges die Nachprüfung durch von der Volksvertretung bestellte +Organe und Hilfsorgane tritt. +% +Grundgesetz Artikel 11 +(1) Alle Deutschen genießen Freizügigkeit im ganzen Bundesgebiet. +(2) Dieses Recht darf nur durch Gesetz oder auf Grund eines Gesetzes +und nur für die Fälle eingeschränkt werden, in denen eine ausreichende +Lebensgrundlage nicht vorhanden ist und der Allgemeinheit daraus +besondere Lasten entstehen würden oder in denen es zur Abwehr einer +drohenden Gefahr für den Bestand oder die freiheitliche demokratische +Grundordnung des Bundes oder eines Landes, zur Bekämpfung von +Seuchengefahr, Naturkatastrophen oder besonders schweren +Unglücksfällen, zum Schutze der Jugend vor Verwahrlosung oder um +strafbaren Handlungen vorzubeugen, erforderlich ist. +% +Grundgesetz Artikel 12 +(1) Alle Deutschen haben das Recht, Beruf, Arbeitsplatz und +Ausbildungsstätte frei zu wählen. Die Berufsausübung kann durch Gesetz +oder auf Grund eines Gesetzes geregelt werden. +(2) Niemand darf zu einer bestimmten Arbeit gezwungen werden, außer im +Rahmen einer herkömmlichen allgemeinen, für alle gleichen öffentlichen +Dienstleistungspflicht. +(3) Zwangsarbeit ist nur bei einer gerichtlich angeordneten +Freiheitsentziehung zulässig. +% +Grundgesetz Artikel 12a +(1) Männer können vom vollendeten achtzehnten Lebensjahr an zum Dienst +in den Streitkräften, im Bundesgrenzschutz oder in einem +Zivilschutzverband verpflichtet werden. +(2) Wer aus Gewissensgründen den Kriegsdienst mit der Waffe +verweigert, kann zu einem Ersatzdienst verpflichtet werden. Die Dauer +des Ersatzdienstes darf die Dauer des Wehrdienstes nicht +übersteigen. Das Nähere regelt ein Gesetz, das die Freiheit der +Gewissensentscheidung nicht beeinträchtigen darf und auch eine +Möglichkeit des Ersatzdienstes vorsehen muß, die in keinem +Zusammenhang mit den Verbänden der Streitkräfte und des +Bundesgrenzschutzes steht. +(3) Wehrpflichtige, die nicht zu einem Dienst nach Absatz 1 oder 2 +herangezogen sind, können im Verteidigungsfalle durch Gesetz oder auf +Grund eines Gesetzes zu zivilen Dienstleistungen für Zwecke der +Verteidigung einschließlich des Schutzes der Zivilbevölkerung in +Arbeitsverhältnisse verpflichtet werden; Verpflichtungen in +öffentlich-rechtliche Dienstverhältnisse sind nur zur Wahrnehmung +polizeilicher Aufgaben oder solcher hoheitlichen Aufgaben der +öffentlichen Verwaltung, die nur in einem öffentlich-rechtlichen +Dienstverhältnis erfüllt werden können, zulässig. Arbeitsverhältnisse +nach Satz 1 können bei den Streitkräften, im Bereich ihrer Versorgung +sowie bei der öffentlichen Verwaltung begründet werden; +Verpflichtungen in Arbeitsverhältnisse im Bereiche der Versorgung der +Zivilbevölkerung sind nur zulässig, um ihren lebensnotwendigen Bedarf +zu decken oder ihren Schutz sicherzustellen. +(4) Kann im Verteidigungsfalle der Bedarf an zivilen Dienstleistungen +im zivilen Sanitäts- und Heilwesen sowie in der ortsfesten +militärischen Lazarettorganisation nicht auf freiwilliger Grundlage +gedeckt werden, so können Frauen vom vollendeten achtzehnten bis zum +vollendeten fünfundfünfzigsten Lebensjahr durch Gesetz oder auf Grund +eines Gesetzes zu derartigen Dienstleistungen herangezogen werden. Sie +dürfen auf keinen Fall zum Dienst mit der Waffe verpflichtet werden. +(5) Für die Zeit vor dem Verteidigungsfalle können Verpflichtungen +nach Absatz 3 nur nach Maßgabe des Artikels 80a Abs. 1 begründet +werden. Zur Vorbereitung auf Dienstleistungen nach Absatz 3, für die +besondere Kenntnisse oder Fertigkeiten erforderlich sind, kann durch +Gesetz oder auf Grund eines Gesetzes die Teilnahme an +Ausbildungsveranstaltungen zur Pflicht gemacht werden. Satz 1 findet +insoweit keine Anwendung. +(6) Kann im Verteidigungsfalle der Bedarf an Arbeitskräften für die in +Absatz 3 Satz 2 genannten Bereiche auf freiwilliger Grundlage nicht +gedeckt werden, so kann zur Sicherung dieses Bedarfs die Freiheit der +Deutschen, die Ausübung eines Berufs oder den Arbeitsplatz aufzugeben, +durch Gesetz oder auf Grund eines Gesetzes eingeschränkt werden. Vor +Eintritt des Verteidigungsfalles gilt Absatz 5 Satz 1 entsprechend. +% +Grundgesetz Artikel 13 +(1) Die Wohnung ist unverletzlich. +(2) Durchsuchungen dürfen nur durch den Richter, bei Gefahr im Verzuge +auch durch die in den Gesetzen vorgesehenen anderen Organe angeordnet +und nur in der dort vorgeschriebenen Form durchgeführt werden. +(3) Begründen bestimmte Tatsachen den Verdacht, daß jemand eine durch +Gesetz einzeln bestimmte besonders schwere Straftat begangen hat, so +dürfen zur Verfolgung der Tat auf Grund richterlicher Anordnung +technische Mittel zur akustischen Überwachung von Wohnungen, in denen +der Beschuldigte sich vermutlich aufhält, eingesetzt werden, wenn die +Erforschung des Sachverhalts auf andere Weise unverhältnismäßig +erschwert oder aussichtslos wäre. Die Maßnahme ist zu befristen. Die +Anordnung erfolgt durch einen mit drei Richtern besetzten +Spruchkörper. Bei Gefahr im Verzuge kann sie auch durch einen +einzelnen Richter getroffen werden. +(4) Zur Abwehr dringender Gefahren für die öffentliche Sicherheit, +insbesondere einer gemeinen Gefahr oder einer Lebensgefahr, dürfen +technische Mittel zur Überwachung von Wohnungen nur auf Grund +richterlicher Anordnung eingesetzt werden. Bei Gefahr im Verzuge kann +die Maßnahme auch durch eine andere gesetzlich bestimmte Stelle +angeordnet werden; eine richterliche Entscheidung ist unverzüglich +nachzuholen. +(5) Sind technische Mittel ausschließlich zum Schutze der bei einem +Einsatz in Wohnungen tätigen Personen vorgesehen, kann die Maßnahme +durch eine gesetzlich bestimmte Stelle angeordnet werden. Eine +anderweitige Verwertung der hierbei erlangten Erkenntnisse ist nur zum +Zwecke der Strafverfolgung oder der Gefahrenabwehr und nur zulässig, +wenn zuvor die Rechtmäßigkeit der Maßnahme richterlich festgestellt +ist; bei Gefahr im Verzuge ist die richterliche Entscheidung +unverzüglich nachzuholen. +(6) Die Bundesregierung unterrichtet den Bundestag jährlich über den +nach Absatz 3 sowie über den im Zuständigkeitsbereich des Bundes nach +Absatz 4 und, soweit richterlich überprüfungsbedürftig, nach Absatz 5 +erfolgten Einsatz technischer Mittel. Ein vom Bundestag gewähltes +Gremium übt auf der Grundlage dieses Berichts die parlamentarische +Kontrolle aus. Die Länder gewährleisten eine gleichwertige +parlamentarische Kontrolle. +(7) Eingriffe und Beschränkungen dürfen im übrigen nur zur Abwehr +einer gemeinen Gefahr oder einer Lebensgefahr für einzelne Personen, +auf Grund eines Gesetzes auch zur Verhütung dringender Gefahren für +die öffentliche Sicherheit und Ordnung, insbesondere zur Behebung der +Raumnot, zur Bekämpfung von Seuchengefahr oder zum Schutze gefährdeter +Jugendlicher vorgenommen werden. +% +Grundgesetz Artikel 14 +(1) Das Eigentum und das Erbrecht werden gewährleistet. Inhalt und +Schranken werden durch die Gesetze bestimmt. +(2) Eigentum verpflichtet. Sein Gebrauch soll zugleich dem Wohle der +Allgemeinheit dienen. +(3) Eine Enteignung ist nur zum Wohle der Allgemeinheit zulässig. Sie +darf nur durch Gesetz oder auf Grund eines Gesetzes erfolgen, das Art +und Ausmaß der Entschädigung regelt. Die Entschädigung ist unter +gerechter Abwägung der Interessen der Allgemeinheit und der +Beteiligten zu bestimmen. Wegen der Höhe der Entschädigung steht im +Streitfalle der Rechtsweg vor den ordentlichen Gerichten offen. +% +Grundgesetz Artikel 15 +Grund und Boden, Naturschätze und Produktionsmittel können zum Zwecke +der Vergesellschaftung durch ein Gesetz, das Art und Ausmaß der +Entschädigung regelt, in Gemeineigentum oder in andere Formen der +Gemeinwirtschaft überführt werden. Für die Entschädigung gilt Artikel +14 Abs. 3 Satz 3 und 4 entsprechend. +% +Grundgesetz Artikel 16 +(1) Die deutsche Staatsangehörigkeit darf nicht entzogen werden. Der +Verlust der Staatsangehörigkeit darf nur auf Grund eines Gesetzes und +gegen den Willen des Betroffenen nur dann eintreten, wenn der +Betroffene dadurch nicht staatenlos wird. +(2) Kein Deutscher darf an das Ausland ausgeliefert werden. Durch +Gesetz kann eine abweichende Regelung für Auslieferungen an einen +Mitgliedstaat der Europäischen Union oder an einen internationalen +Gerichtshof getroffen werden, soweit rechtsstaatliche Grundsätze +gewahrt sind. +% +Grundgesetz Artikel 16a +(1) Politisch Verfolgte genießen Asylrecht. +(2) Auf Absatz 1 kann sich nicht berufen, wer aus einem Mitgliedstaat +der Europäischen Gemeinschaften oder aus einem anderen Drittstaat +einreist, in dem die Anwendung des Abkommens über die Rechtsstellung +der Flüchtlinge und der Konvention zum Schutze der Menschenrechte und +Grundfreiheiten sichergestellt ist. Die Staaten außerhalb der +Europäischen Gemeinschaften, auf die die Voraussetzungen des Satzes 1 +zutreffen, werden durch Gesetz, das der Zustimmung des Bundesrates +bedarf, bestimmt. In den Fällen des Satzes 1 können +aufenthaltsbeendende Maßnahmen unabhängig von einem hiergegen +eingelegten Rechtsbehelf vollzogen werden. +(3) Durch Gesetz, das der Zustimmung des Bundesrates bedarf, können +Staaten bestimmt werden, bei denen auf Grund der Rechtslage, der +Rechtsanwendung und der allgemeinen politischen Verhältnisse +gewährleistet erscheint, daß dort weder politische Verfolgung noch +unmenschliche oder erniedrigende Bestrafung oder Behandlung +stattfindet. Es wird vermutet, daß ein Ausländer aus einem solchen +Staat nicht verfolgt wird, solange er nicht Tatsachen vorträgt, die +die Annahme begründen, daß er entgegen dieser Vermutung politisch +verfolgt wird. +(4) Die Vollziehung aufenthaltsbeendender Maßnahmen wird in den Fällen +des Absatzes 3 und in anderen Fällen, die offensichtlich unbegründet +sind oder als offensichtlich unbegründet gelten, durch das Gericht nur +ausgesetzt, wenn ernstliche Zweifel an der Rechtmäßigkeit der Maßnahme +bestehen; der Prüfungsumfang kann eingeschränkt werden und verspätetes +Vorbringen unberücksichtigt bleiben. Das Nähere ist durch Gesetz zu +bestimmen. +(5) Die Absätze 1 bis 4 stehen völkerrechtlichen Verträgen von +Mitgliedstaaten der Europäischen Gemeinschaften untereinander und mit +dritten Staaten nicht entgegen, die unter Beachtung der +Verpflichtungen aus dem Abkommen über die Rechtsstellung der +Flüchtlinge und der Konvention zum Schutze der Menschenrechte und +Grundfreiheiten, deren Anwendung in den Vertragsstaaten sichergestellt +sein muß, Zuständigkeitsregelungen für die Prüfung von Asylbegehren +einschließlich der gegenseitigen Anerkennung von Asylentscheidungen +treffen. +% +Grundgesetz Artikel 17 +Jedermann hat das Recht, sich einzeln oder in Gemeinschaft mit anderen +schriftlich mit Bitten oder Beschwerden an die zuständigen Stellen und +an die Volksvertretung zu wenden. +% +Grundgesetz Artikel 17a +(1) Gesetze über Wehrdienst und Ersatzdienst können bestimmen, daß für +die Angehörigen der Streitkräfte und des Ersatzdienstes während der +Zeit des Wehr- oder Ersatzdienstes das Grundrecht, seine Meinung in +Wort, Schrift und Bild frei zu äußern und zu verbreiten (Artikel 5 +Abs. 1 Satz 1 erster Halbsatz), das Grundrecht der +Versammlungsfreiheit (Artikel 8) und das Petitionsrecht (Artikel 17), +soweit es das Recht gewährt, Bitten oder Beschwerden in Gemeinschaft +mit anderen vorzubringen, eingeschränkt werden. +(2) Gesetze, die der Verteidigung einschließlich des Schutzes der +Zivilbevölkerung dienen, können bestimmen, daß die Grundrechte der +Freizügigkeit (Artikel 11) und der Unverletzlichkeit der Wohnung +(Artikel 13) eingeschränkt werden. +% +Grundgesetz Artikel 18 +Wer die Freiheit der Meinungsäußerung, insbesondere die Pressefreiheit +(Artikel 5 Abs. 1), die Lehrfreiheit (Artikel 5 Abs. 3), die +Versammlungsfreiheit (Artikel 8), die Vereinigungsfreiheit (Artikel +9), das Brief-, Post- und Fernmeldegeheimnis (Artikel 10), das +Eigentum (Artikel 14) oder das Asylrecht (Artikel 16a) zum Kampfe +gegen die freiheitliche demokratische Grundordnung mißbraucht, +verwirkt diese Grundrechte. Die Verwirkung und ihr Ausmaß werden durch +das Bundesverfassungsgericht ausgesprochen. +% +Grundgesetz Artikel 19 +(1) Soweit nach diesem Grundgesetz ein Grundrecht durch Gesetz oder +auf Grund eines Gesetzes eingeschränkt werden kann, muß das Gesetz +allgemein und nicht nur für den Einzelfall gelten. Außerdem muß das +Gesetz das Grundrecht unter Angabe des Artikels nennen. +(2) In keinem Falle darf ein Grundrecht in seinem Wesensgehalt +angetastet werden. +(3) Die Grundrechte gelten auch für inländische juristische Personen, +soweit sie ihrem Wesen nach auf diese anwendbar sind. +(4) Wird jemand durch die öffentliche Gewalt in seinen Rechten +verletzt, so steht ihm der Rechtsweg offen. Soweit eine andere +Zuständigkeit nicht begründet ist, ist der ordentliche Rechtsweg +gegeben. Artikel 10 Abs. 2 Satz 2 bleibt unberührt. +% -- 2.11.0 From ed1c4664983aa5003eb24950917e5c0999370a16 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 13:16:50 +0200 Subject: [PATCH 164/255] share/skel/dot.profile: Change fortune file to "grundrechte" Obtained from: ElectroBSD --- share/skel/dot.profile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/skel/dot.profile b/share/skel/dot.profile index ad66198ce5e8..eb53895f9097 100644 --- a/share/skel/dot.profile +++ b/share/skel/dot.profile @@ -21,4 +21,4 @@ PAGER=more; export PAGER # set ENV to a file invoked each time sh is started for interactive use. ENV=$HOME/.shrc; export ENV -if [ -x /usr/bin/fortune ] ; then /usr/bin/fortune freebsd-tips ; fi +if [ -x /usr/bin/fortune ] ; then /usr/bin/fortune grundrechte ; fi -- 2.11.0 From a5eaf21c06e2718d0fdab4d27788ad7b4635ca3a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 18 Aug 2015 09:57:30 +0200 Subject: [PATCH 165/255] XEN: Note that ElectroBSD does not protect against compromised hosts Obtained from: ElectroBSD --- sys/x86/xen/hvm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c index e10659e64ad1..49c52806822d 100644 --- a/sys/x86/xen/hvm.c +++ b/sys/x86/xen/hvm.c @@ -335,6 +335,10 @@ xen_hvm_init(enum xen_hvm_init_type init_type) panic("Unsupported HVM initialization type"); } + printf("ElectroBSD security advise: It looks like you may be clown computing.\n" + "Please note that the people controlling the 'cloud' this system is\n" + "running on can monitor and sabotage everything it is doing.\n"); + xen_vector_callback_enabled = 0; xen_hvm_set_callback(NULL); -- 2.11.0 From b2f13fbdc0c2d6e89a431de2497275c9427f9a10 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 12:34:53 +0200 Subject: [PATCH 166/255] sys/vm: vm_pageout_scan: Stop pass after reaching the "maximum" value ... explicitly understood by vm_pageout_scan(). Should prevent (purely cosmetic) issues like: fk@r500 ~ $sudo /usr/src/share/dtrace/monitor-page-scanner [...] 2015 Sep 22 12:15:54: Scan goal 59: Invalid 2015 Sep 22 12:15:54: Scan goal 60: Invalid 2015 Sep 22 12:15:55: Scan goal 61: Invalid 2015 Sep 22 12:15:55: Scan goal 62: Invalid 2015 Sep 22 12:15:55: Scan goals in the previous minute: Launder dirty pages 1 Pageout dirty pages 1 Move inactive to cache or free 2 Invalid 5 Update active LRU/deactivate pages 28 2015 Sep 22 12:15:55: Seconds since last 'Move inactive to cache or free' pass: 30 2015 Sep 22 12:15:55: Seconds since last 'Launder dirty pages' pass: 30 2015 Sep 22 12:15:55: Seconds since last 'Pageout dirty pages' pass: 30 Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index a3f8774c9160..67cf415d2e1e 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -839,6 +839,7 @@ unlock_mp: return (error); } +#define VMD_PASS_MAX 3 /* * vm_pageout_scan does the dirty work for the pageout daemon. * @@ -861,6 +862,9 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) int starting_page_shortage, vnodes_skipped; boolean_t pageout_ok, queue_locked; + KASSERT(pass <= VMD_PASS_MAX, + ("vm_pageout_scan: Invalid pass code %d", pass)); + /* * If we need to reclaim memory ask kernel caches to return * some. We rate limit to avoid thrashing. @@ -1577,7 +1581,8 @@ vm_pageout_worker(void *arg) mtx_unlock(&vm_page_queue_free_mtx); if (pass > 1) pause("psleep", hz / 2); - pass++; + if (pass < VMD_PASS_MAX) + pass++; } else { /* * Yes. Sleep until pages need to be reclaimed or -- 2.11.0 From a708024816f70fed1de270c89a83c5b910bbac11 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 19:10:51 +0200 Subject: [PATCH 167/255] share/dtrace: Add monitor-page-scanner ... which, who would have guessed it, monitors the vm page scanner. It's useful to tune the sysctls for the inactive page limit. Obtained from: ElectroBSD --- share/dtrace/monitor-page-scanner | 168 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 share/dtrace/monitor-page-scanner diff --git a/share/dtrace/monitor-page-scanner b/share/dtrace/monitor-page-scanner new file mode 100755 index 000000000000..85537caff8bf --- /dev/null +++ b/share/dtrace/monitor-page-scanner @@ -0,0 +1,168 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * monitor-page-scanner + * + * Traces the vm page scanner. + * + * Relies on SDT probes that currrently are not part of vanilla FreeBSD. + * + * Copyright (c) 2015 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + goal[0] = "Update active LRU/deactivate pages"; + goal[1] = "Move inactive to cache or free"; + goal[2] = "Launder dirty pages"; + goal[3] = "Pageout dirty pages"; + start_time = walltimestamp; + + min_pass = 2; + + printf("%Y: Monitoring the page scanner. Minimum pass value to show 'boring' scans ", walltimestamp); + printf("without memory pressure or inactive page surplus: %d (%s). Press CTRL-C to abort.\n", + min_pass, goal[min_pass]); +} + +vm:kernel::checked-inactive-pages +{ + this->to_free = (int)arg0; +} + +vm:kernel::checked-inactive-pages +/this->to_free/ +{ + this->above_limit = (int)arg1; + + printf("%Y: %s: Inactive page surplus: %d, above limit: %d.\n", + walltimestamp, probename, this->to_free, this->above_limit); +} + +vm:kernel::before-inactive-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; + this->inactive_page_surplus = (int)arg4; + + /* + * Every pass code above 3 is treated like 3, + * adjust index accordingly. + */ + this->goal_index = (this->pass <= 3) ? this->pass : 3; + this->goal = goal[this->goal_index]; + @goals[this->goal] = count(); + @goals_total[this->goal] = count(); + last_pass[this->pass] = timestamp; +} + +vm:kernel::before-inactive-scan +/(this->pass >= min_pass) || (this->page_shortage > 0) +|| (this->inactive_page_surplus > 0)/ +{ + /* this->vmd = (struct vm_domain *)arg0; */ + this->deficit = (int)arg3; + + printf("%Y: %s: Scan goal %d: %s.\n", + walltimestamp, probename, this->pass, this->goal); + printf("%Y: %s: Page shortage: %d, inactive page surplus: %d\n", + walltimestamp, probename, this->page_shortage, this->inactive_page_surplus); + printf("%Y: v_free_target: %d, v_free_count: %d, v_cache_count: %d.\n", + walltimestamp, `vm_cnt.v_free_target, `vm_cnt.v_free_count, `vm_cnt.v_cache_count); + printf("%Y: v_inactive_target: %d. v_inactive_count: %d. deficit: %d\n", + walltimestamp, `vm_cnt.v_inactive_target, `vm_cnt.v_inactive_count, this->deficit); +} + +vm:kernel::after-inactive-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; + this->addl_page_shortage = (int)arg3; + this->vnodes_skipped = (int)arg4; + this->inactive_page_surplus = (int)arg5; +} + +vm:kernel::after-inactive-scan +/(this->pass >= min_pass) || (this->page_shortage > 0) || +(this->addl_page_shortage > 0) || (this->vnodes_skipped > 0) || +(this->inactive_page_surplus > 0)/ +{ + printf("%Y: %s pass %d: page shortage: %d, inactive page surplus: %d, addl shortage: %d, vnodes skipped: %d.\n", + walltimestamp, probename, this->pass, this->page_shortage, + this->inactive_page_surplus, this->addl_page_shortage, this->vnodes_skipped); +} + +vm:kernel::before-active-scan, +vm:kernel::after-active-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; +} + +vm:kernel::before-active-scan, +vm:kernel::after-active-scan +/(this->pass >= min_pass) || (this->page_shortage > 0)/ +{ + printf("%Y: %s pass %d: page_shortage: %d.\n", + walltimestamp, probename, this->pass, this->page_shortage); +} + +tick-60s +{ + printf("%Y: Scan goals in the previous minute:", walltimestamp); + printa(@goals); + trunc(@goals); +} + +tick-60s +/last_pass[1]/ +{ + this->pass = 1; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +tick-60s +/last_pass[2]/ +{ + this->pass = 2; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +tick-60s +/last_pass[3]/ +{ + this->pass = 3; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +END +{ + printf("%Y: Scan goals since start of script at %Y:", + walltimestamp, start_time); + printa(@goals_total); + + /* Clear aggregates so DTrace does not show them again. */ + trunc(@goals_total); + trunc(@goals); +} -- 2.11.0 From dc2d2ecb65cb6b66cf83e8befaba86f4310a1571 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 13 Oct 2015 18:19:42 +0200 Subject: [PATCH 168/255] libdtrace: Replace another timestamp related assert() with a warning It triggered a lot less often, but was still annoying. Example: 2015 Oct 13 18:01:06: v_free_target: 20888, v_free_count: 44001, v_cache_count: 0. 2015 Oct 13 18:01:06: v_inactive_target: 31332. v_inactive_count: 364311. deficit: 0 2015 Oct 13 18:01:06: after-inactive-scan pass 3: page shortage: -23113, inactive page surplus: 10000, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:06: before-active-scan pass 3: page_shortage: -356092. 2015 Oct 13 18:01:06: after-active-scan pass 3: page_shortage: -356092. 2015 Oct 13 18:01:06: checked-inactive-pages: Inactive page surplus: 10000, above limit: 252979. dtrace: cpu clocks out of sync (18128465664478 < 18128471798881; offset: 6134403). Results may be incorrect! dtrace: cpu clocks out of sync (18129463671844 < 18129469778266; offset: 6106422). Results may be incorrect! 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253446. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253446. 2015 Oct 13 18:01:07: before-inactive-scan: Scan goal 1: Move inactive to cache or free. 2015 Oct 13 18:01:07: before-inactive-scan: Page shortage: -37096, inactive page surplus: 10000 2015 Oct 13 18:01:07: v_free_target: 20888, v_free_count: 57984, v_cache_count: 0. 2015 Oct 13 18:01:07: v_inactive_target: 31332. v_inactive_count: 364778. deficit: 0 2015 Oct 13 18:01:07: after-inactive-scan pass 1: page shortage: -37203, inactive page surplus: 9893, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253250. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253250. 2015 Oct 13 18:01:07: before-inactive-scan: Scan goal 2: Launder dirty pages. 2015 Oct 13 18:01:07: before-inactive-scan: Page shortage: -37191, inactive page surplus: 10000 2015 Oct 13 18:01:07: v_free_target: 20888, v_free_count: 58079, v_cache_count: 0. 2015 Oct 13 18:01:07: v_inactive_target: 31332. v_inactive_count: 364582. deficit: 0 2015 Oct 13 18:01:07: after-inactive-scan pass 2: page shortage: -37194, inactive page surplus: 9997, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:07: before-active-scan pass 2: page_shortage: -370441. 2015 Oct 13 18:01:07: after-active-scan pass 2: page_shortage: -370441. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253268. dtrace: cpu clocks out of sync (18130462652022 < 18130468761237; offset: 6109215). Results may be incorrect! 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253331. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253331. 2015 Oct 13 18:01:08: before-inactive-scan: Scan goal 1: Move inactive to cache or free. 2015 Oct 13 18:01:08: before-inactive-scan: Page shortage: -34625, inactive page surplus: 10000 2015 Oct 13 18:01:08: v_free_target: 20888, v_free_count: 55513, v_cache_count: 0. 2015 Oct 13 18:01:08: v_inactive_target: 31332. v_inactive_count: 364663. deficit: 0 2015 Oct 13 18:01:08: after-inactive-scan pass 1: page shortage: -34645, inactive page surplus: 9980, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253286. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253286. 2015 Oct 13 18:01:08: before-inactive-scan: Scan goal 2: Launder dirty pages. 2015 Oct 13 18:01:08: before-inactive-scan: Page shortage: -34395, inactive page surplus: 10000 2015 Oct 13 18:01:08: v_free_target: 20888, v_free_count: 55283, v_cache_count: 0. 2015 Oct 13 18:01:08: v_inactive_target: 31332. v_inactive_count: 364618. deficit: 0 2015 Oct 13 18:01:08: after-inactive-scan pass 2: page shortage: -34396, inactive page surplus: 9999, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:08: before-active-scan pass 2: page_shortage: -367391. 2015 Oct 13 18:01:08: after-active-scan pass 2: page_shortage: -367391. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253285. 2015 Oct 13 18:01:09: checked-inactive-pages: Inactive page surplus: 10000, above limit: 256158. 2015 Oct 13 18:01:09: before-inactive-scan: Scan goal 3: Pageout dirty pages. 2015 Oct 13 18:01:09: before-inactive-scan: Page shortage: -34665, inactive page surplus: 10000 2015 Oct 13 18:01:09: v_free_target: 20888, v_free_count: 55553, v_cache_count: 0. 2015 Oct 13 18:01:09: v_inactive_target: 31332. v_inactive_count: 367490. deficit: 0 dtrace: cpu clocks out of sync (18131462608939 < 18131468746340; offset: 6137401). Results may be incorrect! Assertion failed: (timestamp >= dtp->dt_last_timestamp), file /usr/src/cddl/lib/libdtrace/../../../cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c, line 3352. Obtained from: ElectroBSD --- cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c index 39530cf41e83..3b66c3522cdc 100644 --- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c +++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c @@ -3349,7 +3349,13 @@ dtrace_consume(dtrace_hdl_t *dtp, FILE *fp, break; timestamp = dt_buf_oldest(buf, dtp); - assert(timestamp >= dtp->dt_last_timestamp); + if (timestamp < dtp->dt_last_timestamp) { + warnx("cpu clocks out of sync " + "(%ju < %ju; offset: %ju). " + "Results may be incorrect!", + timestamp, dtp->dt_last_timestamp, + dtp->dt_last_timestamp - timestamp); + } dtp->dt_last_timestamp = timestamp; if (timestamp == buf->dtbd_timestamp) { -- 2.11.0 From 20848d39d153d152852952740555465c64bfae86 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 24 Oct 2015 12:58:42 +0200 Subject: [PATCH 169/255] src.opts.mk: Disable EXAMPLES so we don't have to care about license issues Obtained from: ElectroBSD --- share/mk/src.opts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index d4bd454b462f..6e12a2118f64 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -78,7 +78,6 @@ __DEFAULT_YES_OPTIONS = \ EE \ ELFCOPY_AS_OBJCOPY \ ELFTOOLCHAIN_BOOTSTRAP \ - EXAMPLES \ FDT \ FILE \ FINGER \ @@ -174,6 +173,7 @@ __DEFAULT_NO_OPTIONS += \ BSDINSTALL \ CCD \ CTM \ + EXAMPLES \ FLOPPY \ FMTREE \ FREEBSD_UPDATE \ -- 2.11.0 From 1ac08b2c547e320c75d11c444165efb4d961434c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 13:49:38 +0200 Subject: [PATCH 170/255] sys/modules: Don't (try to) build qlxbge if the user objects to binary blobs The module depends on source-less microcode in sys/dev/qlxgbe/ql_fw.c. Obtained from: ElectroBSD --- sys/modules/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 0795a902a43e..11a9e3d3a7e0 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -682,7 +682,9 @@ _ntb= ntb _pms= pms _qlxge= qlxge _qlxgb= qlxgb +.if ${MK_SOURCELESS_UCODE} != "no" _qlxgbe= qlxgbe +.endif _sfxge= sfxge .if ${MK_BHYVE} != "no" || defined(ALL_MODULES) -- 2.11.0 From 69cbd53667c280113cd66bb19055068534917bf0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 12:52:29 +0200 Subject: [PATCH 171/255] sys/modules: Don't (try to) build bxe if the user objects to binary blobs The module contains source-less microcode: sys/dev/bxe/5771[0-2]_init_values.c Obtained from: ElectroBSD --- sys/modules/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 11a9e3d3a7e0..eafbe5bcf7ba 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -543,7 +543,9 @@ _an= an _aout= aout _bios= bios _bktr= bktr +.if ${MK_SOURCELESS_UCODE} != "no" _bxe= bxe +.endif _cardbus= cardbus _cbb= cbb _cpuctl= cpuctl -- 2.11.0 From 8a629a5e11b9f252109412ec5c97fe2ba32680e6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 27 Nov 2015 00:20:59 +0100 Subject: [PATCH 172/255] etc: Register uid and gid for Tor ... so the user doesn't have to do it manually on the host system after installing Tor in a jail. Obtained from: ElectroBSD --- etc/group | 1 + etc/master.passwd | 1 + 2 files changed, 2 insertions(+) diff --git a/etc/group b/etc/group index 3136230b0790..49eccdcb8dac 100644 --- a/etc/group +++ b/etc/group @@ -30,6 +30,7 @@ network:*:69: audit:*:77: www:*:80: _ypldap:*:160: +_tor:*:256: hast:*:845: ggatec:*:846: ggated:*:847: diff --git a/etc/master.passwd b/etc/master.passwd index c6005dc66899..b4a4a9b23f0e 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -23,6 +23,7 @@ pop:*:68:6::0:0:Post Office Owner:/nonexistent:/usr/sbin/nologin auditdistd:*:78:77::0:0:Auditdistd unprivileged user:/var/empty:/usr/sbin/nologin www:*:80:80::0:0:World Wide Web Owner:/nonexistent:/usr/sbin/nologin _ypldap:*:160:160::0:0:YP LDAP unprivileged user:/var/empty:/usr/sbin/nologin +_tor:*:256:256::0:0:Onion delivery agent:/nonexistent:/usr/sbin/nologin hast:*:845:845::0:0:HAST unprivileged user:/var/empty:/usr/sbin/nologin ggatec:*:846:846::0:0:ggatec unprivileged user:/var/empty:/usr/sbin/nologin ggated:*:847:847::0:0:ggated unprivileged user:/var/empty:/usr/sbin/nologin -- 2.11.0 From eab901f014f5f5258b2d0adf1f5fa6875f663026 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 16 Dec 2015 12:28:43 +0100 Subject: [PATCH 173/255] fork_findpid(): Declare p static to help diagnosing panics Like this one: Fatal trap 12: page fault while in kernel mode cpuid = 1; apic id = 04 fault virtual address = 0x618b00a8 fault code = supervisor read data, page not present instruction pointer = 0x20:0xffffffff80909158 stack pointer = 0x28:0xfffffe011e03b940 frame pointer = 0x28:0xfffffe011e03b960 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 71325 (sh) trap number = 12 panic: page fault cpuid = 1 KDB: stack backtrace: [...] Uptime: 13d20h43m20s [...] (kgdb) where #0 doadump (textdump=1) at pcpu.h:221 #1 0xffffffff8094a923 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:364 #2 0xffffffff8094ae8b in vpanic (fmt=, ap=) at /usr/src/sys/kern/kern_shutdown.c:757 #3 0xffffffff8094acc3 in panic (fmt=0x0) at /usr/src/sys/kern/kern_shutdown.c:688 #4 0xffffffff80c2fbb1 in trap_fatal (frame=, eva=) at /usr/src/sys/amd64/amd64/trap.c:834 #5 0xffffffff80c2fda4 in trap_pfault (frame=0xfffffe011e03b890, usermode=) at /usr/src/sys/amd64/amd64/trap.c:684 #6 0xffffffff80c2f55e in trap (frame=0xfffffe011e03b890) at /usr/src/sys/amd64/amd64/trap.c:435 #7 0xffffffff80c120a7 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:234 #8 0xffffffff80909158 in fork_findpid (flags=) at /usr/src/sys/kern/kern_fork.c:281 #9 0xffffffff80907225 in do_fork (td=0xfffff8009db9a9a0, flags=20, p2=0xfffff8009dbe1a90, td2=0xfffff800aa6884d0, vm2=0xfffff800a9eee000, pdflags=0) at /usr/src/sys/kern/kern_fork.c:385 #10 0xffffffff80906c08 in fork1 (td=0xfffff8009db9a9a0, flags=20, pages=, procp=0xfffffe011e03bac0, procdescp=0x0, pdflags=99999, fcaps=) at /usr/src/sys/kern/kern_fork.c:937 #11 0xffffffff809066ca in sys_fork (td=0xfffff8009db9a9a0, uap=) at /usr/src/sys/kern/kern_fork.c:108 #12 0xffffffff80c3054b in amd64_syscall (td=0xfffff8009db9a9a0, traced=0) at subr_syscall.c:140 #13 0xffffffff80c1238b in Xfast_syscall () at /usr/src/sys/amd64/amd64/exception.S:394 #14 0x00000008009257aa in ?? () Previous frame inner to this frame (corrupt stack?) Current language: auto; currently minimal (kgdb) f 8 #8 0xffffffff80909158 in fork_findpid (flags=) at /usr/src/sys/kern/kern_fork.c:281 warning: Source file is more recent than executable. 281 (p->p_pgrp != NULL && (kgdb) l - 271 * id is kept reserved only while there is a 272 * non-reaped process in the subtree, so amount of 273 * reserved pids is limited by process limit times 274 * two. 275 */ 276 p = LIST_FIRST(&allproc); 277 again: 278 for (; p != NULL; p = LIST_NEXT(p, p_list)) { 279 while (p->p_pid == trypid || 280 p->p_reapsubtree == trypid || (kgdb) l 281 (p->p_pgrp != NULL && 282 (p->p_pgrp->pg_id == trypid || 283 (p->p_session != NULL && 284 p->p_session->s_sid == trypid)))) { 285 trypid++; 286 if (trypid >= pidchecked) 287 goto retry; 288 } 289 if (p->p_pid > trypid && pidchecked > p->p_pid) 290 pidchecked = p->p_pid; (kgdb) p p No symbol "p" in current context. Obtained from: ElectroBSD --- sys/kern/kern_fork.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index ff1c0a0a3744..6949604899cd 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -226,7 +226,7 @@ SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW, static int fork_findpid(int flags) { - struct proc *p; + static struct proc *p; int trypid; static int pidchecked = 0; -- 2.11.0 From ad55725fdb3e70d163c75e4b50d967b764047e9c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 7 Jan 2016 16:58:50 +0100 Subject: [PATCH 174/255] sys/sys/copyright.h: Update copyright Obtained from: ElectroBSD --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index 0e070c1af60c..bf489bed8d55 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -30,7 +30,7 @@ /* Add a FreeBSD vendor copyright here */ #define COPYRIGHT_Vendor \ - "Copyright (c) 2010-2015 Fabian Keil - IT-Beratung und Polizei-Erziehung\n" + "Copyright (c) 2010-2016 Fabian Keil - IT-Beratung und Polizei-Erziehung\n" /* FreeBSD */ #define COPYRIGHT_FreeBSD \ -- 2.11.0 From 3718b150e947c5507367e1199dbe94e7469a7643 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 Jan 2016 00:37:52 +0100 Subject: [PATCH 175/255] crontab: Default to not send out (most likely unencrypted) mails Obtained from: ElectroBSD --- etc/crontab | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/crontab b/etc/crontab index e1e6e8862c8d..22204f81a0fd 100644 --- a/etc/crontab +++ b/etc/crontab @@ -1,9 +1,8 @@ -# /etc/crontab - root's crontab for FreeBSD -# -# $FreeBSD$ +# /etc/crontab - root's crontab for ElectroBSD # SHELL=/bin/sh PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin +MAILTO="" # #minute hour mday month wday who command # -- 2.11.0 From a7ae62203d14f1ccbc9d2a8b22346770ae6961c3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 16 Jan 2016 15:24:51 +0100 Subject: [PATCH 176/255] Detach unreproducible uzip tests (added in r293821) that aren't relevant for ElectroBSD Obtained from: ElectroBSD --- tests/sys/geom/class/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sys/geom/class/Makefile b/tests/sys/geom/class/Makefile index b633fd081ee1..92c9b9f8e38f 100644 --- a/tests/sys/geom/class/Makefile +++ b/tests/sys/geom/class/Makefile @@ -14,7 +14,8 @@ TESTS_SUBDIRS+= nop TESTS_SUBDIRS+= raid3 TESTS_SUBDIRS+= shsec TESTS_SUBDIRS+= stripe -TESTS_SUBDIRS+= uzip +# Currently not reproducible +#TESTS_SUBDIRS+= uzip ${PACKAGE}FILES+= geom_subr.sh -- 2.11.0 From d4371106b037893f9a0a073c6791eeb95b0eaf72 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 Jan 2016 12:49:29 +0100 Subject: [PATCH 177/255] OpenSSH: Disable DSA keys by default ... as it's done by vanilla OpenSSH since 6.9p1. This reverts FreeBSD r294495 which enabled them again due to POLA concerns that aren't relevant for ElectroBSD. Obtained from: ElectroBSD --- UPDATING | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/UPDATING b/UPDATING index b9e1e1770a1c..09ea202f7463 100644 --- a/UPDATING +++ b/UPDATING @@ -182,6 +182,10 @@ from older version of current across the gcc/clang cutover is a bit fragile. restored by setting vfs.zfs.vol.recursive=1. 20160119: + The default configuration of ssh(1) no longer allows to use ssh-dss + keys. To enable using them, add 'ssh-dss' to PubkeyAcceptedKeyTypes + option in the /etc/ssh/ssh_config. Refer to ssh_config(5) for more + information. The NONE and HPN patches has been removed from OpenSSH. They are still available in the security/openssh-portable port. -- 2.11.0 From f099d7c315f8509c96843755c1bd2cca96b3eea4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:25:31 +0100 Subject: [PATCH 178/255] OpenSSH: Disable tcp wrapper support Obtained from: ElectroBSD --- crypto/openssh/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/openssh/config.h b/crypto/openssh/config.h index 364ebe33e2ef..a26c7647c913 100644 --- a/crypto/openssh/config.h +++ b/crypto/openssh/config.h @@ -1408,7 +1408,7 @@ /* #undef LASTLOG_WRITE_PUTUTXLINE */ /* Define if you want TCP Wrappers support */ -#define LIBWRAP 1 +/* #undef LIBWRAP 1 */ /* Define to whatever link() returns for "not supported" if it doesn't return EOPNOTSUPP. */ -- 2.11.0 From 2a79349508618871f624122606c72643742e3963 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:27:33 +0100 Subject: [PATCH 179/255] OpenSSH: Remove AES-CBC ciphers from the default cipher list on the server As it's done in OpenSSH 6.7p1 and later. This reverts FreeBSD r296634. Obtained from: ElectroBSD --- crypto/openssh/FREEBSD-upgrade | 8 +------- crypto/openssh/myproposal.h | 5 ++--- crypto/openssh/sshd_config.5 | 3 +-- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/crypto/openssh/FREEBSD-upgrade b/crypto/openssh/FREEBSD-upgrade index 4b31eb3532d6..8161baddaf5d 100644 --- a/crypto/openssh/FREEBSD-upgrade +++ b/crypto/openssh/FREEBSD-upgrade @@ -1,3 +1,4 @@ + FreeBSD maintainer's guide to OpenSSH-portable ============================================== @@ -160,13 +161,6 @@ ignore HPN-related configuration options to avoid breaking existing configurations. -9) AES-CBC - - The AES-CBC ciphers were removed from the server-side proposal list - in 6.7p1 due to theoretical weaknesses and the availability of - superior ciphers (including AES-CTR and AES-GCM). We have re-added - them for compatibility with third-party clients. - This port was brought to you by (in no particular order) DARPA, NAI diff --git a/crypto/openssh/myproposal.h b/crypto/openssh/myproposal.h index 68d29e86b63e..2af5c47cf82a 100644 --- a/crypto/openssh/myproposal.h +++ b/crypto/openssh/myproposal.h @@ -111,11 +111,10 @@ #define KEX_SERVER_ENCRYPT \ "chacha20-poly1305@openssh.com," \ "aes128-ctr,aes192-ctr,aes256-ctr" \ - AESGCM_CIPHER_MODES \ - ",aes128-cbc,aes192-cbc,aes256-cbc" + AESGCM_CIPHER_MODES #define KEX_CLIENT_ENCRYPT KEX_SERVER_ENCRYPT "," \ - "3des-cbc" + "aes128-cbc,aes192-cbc,aes256-cbc,3des-cbc" #define KEX_SERVER_MAC \ "umac-64-etm@openssh.com," \ diff --git a/crypto/openssh/sshd_config.5 b/crypto/openssh/sshd_config.5 index 54c95ad74220..4591b1f5e583 100644 --- a/crypto/openssh/sshd_config.5 +++ b/crypto/openssh/sshd_config.5 @@ -482,8 +482,7 @@ The default is: .Bd -literal -offset indent chacha20-poly1305@openssh.com, aes128-ctr,aes192-ctr,aes256-ctr, -aes128-gcm@openssh.com,aes256-gcm@openssh.com, -aes128-cbc,aes192-cbc,aes256-cbc +aes128-gcm@openssh.com,aes256-gcm@openssh.com .Ed .Pp The list of available ciphers may also be obtained using the -- 2.11.0 From d3b8fa28df3337f90ea0e4db47bd4d940d751bb3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:53:42 +0100 Subject: [PATCH 180/255] sshd: Disable UsePam, X11Forwarding and UseDNS by default Obtained from: ElectroBSD --- crypto/openssh/servconf.c | 6 +++--- crypto/openssh/sshd_config | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/openssh/servconf.c b/crypto/openssh/servconf.c index 66d24cefd4e2..9844248ef51c 100644 --- a/crypto/openssh/servconf.c +++ b/crypto/openssh/servconf.c @@ -203,7 +203,7 @@ fill_default_server_options(ServerOptions *options) /* Portable-specific options */ if (options->use_pam == -1) - options->use_pam = 1; + options->use_pam = 0; /* Standard Options */ if (options->protocol == SSH_PROTO_UNKNOWN) @@ -256,7 +256,7 @@ fill_default_server_options(ServerOptions *options) if (options->print_lastlog == -1) options->print_lastlog = 1; if (options->x11_forwarding == -1) - options->x11_forwarding = 1; + options->x11_forwarding = 0; if (options->x11_display_offset == -1) options->x11_display_offset = 10; if (options->x11_use_localhost == -1) @@ -336,7 +336,7 @@ fill_default_server_options(ServerOptions *options) if (options->max_sessions == -1) options->max_sessions = DEFAULT_SESSIONS_MAX; if (options->use_dns == -1) - options->use_dns = 1; + options->use_dns = 0; if (options->client_alive_interval == -1) options->client_alive_interval = 0; if (options->client_alive_count_max == -1) diff --git a/crypto/openssh/sshd_config b/crypto/openssh/sshd_config index a842e6efef9a..a22c0d3ec470 100644 --- a/crypto/openssh/sshd_config +++ b/crypto/openssh/sshd_config @@ -97,12 +97,12 @@ # If you just want the PAM account and session checks to run without # PAM authentication, then enable this but set PasswordAuthentication # and ChallengeResponseAuthentication to 'no'. -#UsePAM yes +#UsePAM no #AllowAgentForwarding yes #AllowTcpForwarding yes #GatewayPorts no -#X11Forwarding yes +#X11Forwarding no #X11DisplayOffset 10 #X11UseLocalhost yes #PermitTTY yes @@ -115,7 +115,7 @@ #Compression delayed #ClientAliveInterval 0 #ClientAliveCountMax 3 -#UseDNS yes +#UseDNS no #PidFile /var/run/sshd.pid #MaxStartups 10:30:100 #PermitTunnel no -- 2.11.0 From b497013ab8918736b49a4cbcb4e6e86d5d7b0ebd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:42:00 +0100 Subject: [PATCH 181/255] OpenSSH ssh and sshd: Default to not sending an VersionAddendum This is also the default used by OpenSSH upstream. Obtained from: ElectroBSD --- crypto/openssh/ssh_config | 2 +- crypto/openssh/sshd_config | 2 +- crypto/openssh/version.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/openssh/ssh_config b/crypto/openssh/ssh_config index 8eca3453e298..d7df585bcdfd 100644 --- a/crypto/openssh/ssh_config +++ b/crypto/openssh/ssh_config @@ -50,4 +50,4 @@ # ProxyCommand ssh -q -W %h:%p gateway.example.com # RekeyLimit 1G 1h # VerifyHostKeyDNS yes -# VersionAddendum FreeBSD-20160310 +# VersionAddendum none diff --git a/crypto/openssh/sshd_config b/crypto/openssh/sshd_config index a22c0d3ec470..94b94e8b5d6e 100644 --- a/crypto/openssh/sshd_config +++ b/crypto/openssh/sshd_config @@ -121,7 +121,7 @@ #PermitTunnel no #ChrootDirectory none #UseBlacklist no -#VersionAddendum FreeBSD-20160310 +#VersionAddendum none # no default banner path #Banner none diff --git a/crypto/openssh/version.h b/crypto/openssh/version.h index 031bb87ea6b8..4068d1775594 100644 --- a/crypto/openssh/version.h +++ b/crypto/openssh/version.h @@ -6,7 +6,7 @@ #define SSH_PORTABLE "p2" #define SSH_RELEASE SSH_VERSION SSH_PORTABLE -#define SSH_VERSION_FREEBSD "FreeBSD-20160310" +#define SSH_VERSION_FREEBSD "" #ifdef WITH_OPENSSL #define OPENSSL_VERSION SSLeay_version(SSLEAY_VERSION) -- 2.11.0 From cea34493e0044608a8726d891262056be22563b4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 2 Feb 2016 20:28:35 +0100 Subject: [PATCH 182/255] release/amd64/make-memstick.sh: Use the shiny new -T switch to clamp the remaining timestamps Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 66ee073c63ba..4263855e011d 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -37,7 +37,11 @@ echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local mtree -c -k time -p "${1}" | sed \ -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ > "${2}.mtree" || return 1 -makefs -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} + +# The base makefs may not have -T support yet, +# thus we use the fresh one that goes into the image. +dist/base/usr/sbin/makefs -T "${EPOCH_DATE-0}" \ + -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 -- 2.11.0 From c10b7fbad1d331ad6792a2dea504bcc9ac8ae1bd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Feb 2016 15:38:15 +0100 Subject: [PATCH 183/255] release/scripts/strip-freebsd.sh: Add riscv to the list of unsupported architectures Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index 56be187cca4c..9e49e1fa8cca 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -74,7 +74,7 @@ get_files_to_ditch() { # source tarball and to reduce the number of files that # should be audited for license and security issues. get_unsupported_architectures() { - echo "arm arm64 mips pc98 powerpc sparc64" + echo "arm arm64 mips pc98 powerpc riscv sparc64" } # These depend on or contain proprietary firmware that is included in sys/contrib/dev -- 2.11.0 From 0d1a847faf452bfa1f2fe62925371ca6213f83f0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Feb 2016 15:49:00 +0100 Subject: [PATCH 184/255] strip-freebsd.sh: Delete sys/netnatm and sys/ofed Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index 9e49e1fa8cca..fea0889485a6 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -97,6 +97,11 @@ get_unused_contrib_dirs() { echo "apr apr-util ipfilter ofed sendmail serf subversion tcsh" } +# Only includes directory that aren't architecture-specific +get_unused_cross_platform_sys_dirs() { + echo "netnatm ofed" +} + get_directories_to_ditch() { local arch \ dir arch_dir sys_contrib contrib_dir @@ -110,6 +115,13 @@ get_directories_to_ditch() { done done + for dir in $(get_unused_cross_platform_sys_dirs); do + potential_directory="sys/${dir}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done + for sys_contrib in ipfilter octeon-sdk; do echo "sys/contrib/${sys_contrib}" done -- 2.11.0 From e5322c67565be8f405b989b50cfdb67252319532 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 22 Feb 2016 13:15:19 +0100 Subject: [PATCH 185/255] include/Makefile: Unbreak build without atm sources Obtained from: ElectroBSD --- include/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/Makefile b/include/Makefile index 2fb8e3ca6e8a..48f9f653f932 100644 --- a/include/Makefile +++ b/include/Makefile @@ -40,7 +40,7 @@ LHDRS= aio.h errno.h fcntl.h linker_set.h poll.h stdatomic.h stdint.h \ syslog.h ucontext.h LDIRS= bsm cam geom net net80211 netgraph netinet netinet6 \ - netipsec netnatm netsmb nfs nfsclient nfsserver sys vm + netipsec netsmb nfs nfsclient nfsserver sys vm LSUBDIRS= cam/ata cam/nvme cam/scsi \ dev/acpica dev/agp dev/an dev/bktr dev/ciss dev/filemon dev/firewire \ @@ -54,7 +54,7 @@ LSUBDIRS= cam/ata cam/nvme cam/scsi \ geom/mirror geom/mountver geom/multipath geom/nop \ geom/raid geom/raid3 geom/shsec geom/stripe geom/virstor \ net/altq \ - netgraph/atm netgraph/netflow \ + netgraph/netflow \ netinet/cc \ security/audit \ security/mac_biba security/mac_bsdextended security/mac_lomac \ -- 2.11.0 From 6f1bf60e91351117f655eedffbef2340e77c8cea Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 22 Feb 2016 16:24:43 +0100 Subject: [PATCH 186/255] sys/modules: Skip mlx5 when building without OFED Fixes: ===> mlx (depend) machine -> /usr/src/sys/amd64/include x86 -> /usr/src/sys/x86/include awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/device_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/bus_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/dev/pci/pci_if.m -h rm -f .depend CC='cc' mkdep -f .depend -a -nostdinc -D_KERNEL -DKLD_MODULE -DHAVE_KERNEL_OPTION_HEADERS -I. -I/usr/src/sys -I/usr/obj/usr/src/sys/ELECTRO_BLOAT -D__printf__=__freebsd_kprintf__ -std=iso9899:1999 -include /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_global.h /usr/src/sys/modules/mlx/../../dev/mlx/mlx.c /usr/src/sys/modules/mlx/../../dev/mlx/mlx_pci.c /usr/src/sys/modules/mlx/../../dev/mlx/mlx_disk.c ===> mlx5 (depend) machine -> /usr/src/sys/amd64/include x86 -> /usr/src/sys/x86/include awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/device_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/bus_if.m -h awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -p awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -q awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/dev/pci/pci_if.m -h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet.h opt_inet.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet6.h opt_inet6.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_random.h opt_random.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_rss.h opt_rss.h rm -f .depend CC='cc' mkdep -f .depend -a -nostdinc -D_KERNEL -DKLD_MODULE -I/usr/src/sys/modules/mlx5/../../ofed/include -I/usr/src/sys/modules/mlx5/../../compat/linuxkpi/common/include -DHAVE_KERNEL_OPTION_HEADERS -I. -I /usr/src/sys -I/usr/obj/usr/src/sys/ELECTRO_BLOAT -D__printf__=__freebsd_kprintf__ -std=iso9899:1999 -include /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_global.h /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core /mlx5_alloc.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cmd.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cq.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_eq.c /usr/src/sys /modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_eswitch_vacl.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_flow_table.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_fw.c /usr/src/sys/modules/ mlx5/../../dev/mlx5/mlx5_core/mlx5_health.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_mad.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_main.c /usr/src/sys/modules/mlx5/../../dev/mlx5/ mlx5_core/mlx5_mcg.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_mr.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_pagealloc.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_pd.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_port.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_qp.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_srq.c /usr/src/sys/modules/ml x5/../../dev/mlx5/mlx5_core/mlx5_transobj.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_uar.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_vport.c /usr/src/sys/modules/mlx5/../../dev/mlx5 /mlx5_core/mlx5_wq.c In file included from /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_alloc.c:33: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. In file included from /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cmd.c:38: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. In file included from /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cq.c:31: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. [...] Obtained from: ElectroBSD --- sys/modules/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index eafbe5bcf7ba..8a4f794c6132 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -639,8 +639,8 @@ _iwnfw= iwnfw _mlx4= mlx4 _mlx4ib= mlx4ib _mlxen= mlxen -.endif _mlx5= mlx5 +.endif .if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _mlx5en= mlx5en -- 2.11.0 From eaf922a7ee8327f1c35971e4a6417e4de505efe9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 22 Feb 2016 17:03:33 +0100 Subject: [PATCH 187/255] sys/modules: Skip mlx5en when building without OFED Fixes: ===> mlx5en (all) machine -> /usr/src/sys/amd64/include x86 -> /usr/src/sys/x86/include awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/device_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/bus_if.m -h awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -p awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -q awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/dev/pci/pci_if.m -h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet.h opt_inet.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet6.h opt_inet6.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_rss.h opt_rss.h cc -O2 -pipe -fno-strict-aliasing -Werror -D_KERNEL -DKLD_MODULE -nostdinc -I/usr/src/sys/modules/mlx5en/../../ofed/include -I/usr/src/sys/modules/mlx5en/../../compat/linuxkpi/common/include -DHAVE_KERNEL_OPTION_HEADERS -include /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_global.h -I. -I/usr/src/sys -fno-common -g -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -I/usr/obj/usr/src/sys/ELECTRO_BLOAT -mcmodel=kernel -mno-red-zone -mno-mmx -mno-sse -msoft-float -fno-asynchronous-unwind-tables -ffreestanding -fwrapv -fstack-protector -gdwarf-2 -Wall -Wredundant-decls -Wnested-externs -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith -Winline -Wcast-qual -Wundef -Wno-pointer-sign -D__printf__=__freebsd_kprintf__ -Wmissing-include-dirs -fdiagnostics-show-option -Wno-unknown-pragmas -Wno-error-tautological-compare -Wno-error-empty-body -Wno-error-parentheses-equality -Wno-error-unused-function -Wno-error-pointer-sign -Wno-error-shift-negative-value -mno-aes -mno-avx -std=iso9899:1999 -Wno-cast-qual -Wno-pointer-arith -c /usr/src/sys/modules/mlx5en/../../dev/mlx5/mlx5_en/mlx5_en_ethtool.c -o mlx5_en_ethtool.o In file included from /usr/src/sys/modules/mlx5en/../../dev/mlx5/mlx5_en/mlx5_en_ethtool.c:28: In file included from /usr/src/sys/modules/mlx5en/../../dev/mlx5/mlx5_en/en.h:66: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. *** Error code 1 Stop. make[4]: stopped in /usr/src/sys/modules/mlx5en *** Error code 1 Stop. make[3]: stopped in /usr/src/sys/modules *** Error code 1 Stop. make[2]: stopped in /usr/obj/usr/src/sys/ELECTRO_BLOAT *** Error code 1 Stop. make[1]: stopped in /usr/src *** Error code 1 Stop. make: stopped in /usr/src Obtained from: ElectroBSD --- sys/modules/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 8a4f794c6132..dda252f4e7ec 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -640,11 +640,11 @@ _mlx4= mlx4 _mlx4ib= mlx4ib _mlxen= mlxen _mlx5= mlx5 -.endif .if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _mlx5en= mlx5en .endif +.endif _mly= mly .if ${MK_OFED} != "no" || defined(ALL_MODULES) _mthca= mthca -- 2.11.0 From b20ee403b174a2a1abf421d99f6bee0d2445d135 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 29 Feb 2016 09:12:05 +0100 Subject: [PATCH 188/255] release/scripts/strip-freebsd.sh: Ditch ncsw which was added in r296177/8bf1194fe5 Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index fea0889485a6..beb9128af517 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -122,7 +122,7 @@ get_directories_to_ditch() { fi done - for sys_contrib in ipfilter octeon-sdk; do + for sys_contrib in ipfilter ncsw octeon-sdk; do echo "sys/contrib/${sys_contrib}" done -- 2.11.0 From 9cd7864cd8840f40610f8c3ae4b9b7cbba652d19 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Mar 2016 12:51:02 +0100 Subject: [PATCH 189/255] sys/cam: Retry in case of 'uncorrectable' errors At least in case of the LITE-ON DVDRW SOHW-1693S with firmware KC4B, they are recoverable. Maybe this should be done as device-specific quirk. Obtained from: ElectroBSD --- sys/cam/scsi/scsi_all.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c index 29e5ac68df8d..fde1ab4b7c9a 100644 --- a/sys/cam/scsi/scsi_all.c +++ b/sys/cam/scsi/scsi_all.c @@ -1407,8 +1407,8 @@ static struct asc_table_entry asc_table[] = { { SST(0x11, 0x04, SS_FATAL|EIO, "Unrecovered read error - auto reallocate failed") }, /* WRO B */ - { SST(0x11, 0x05, SS_FATAL|EIO, - "L-EC uncorrectable error") }, + { SST(0x11, 0x05, SS_RDEF, + "L-EC 'uncorrectable' error") }, /* WRO B */ { SST(0x11, 0x06, SS_FATAL|EIO, "CIRC unrecovered error") }, -- 2.11.0 From d0e3708c8296768760c3f012a371b5321a9d1886 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 16:54:42 +0100 Subject: [PATCH 190/255] sys/netinet: Default to not processing IP options Obtained from: ElectroBSD --- sys/netinet/ip_options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c index 3e44ffb7b7c9..cc640d1e16bf 100644 --- a/sys/netinet/ip_options.c +++ b/sys/netinet/ip_options.c @@ -78,7 +78,7 @@ SYSCTL_INT(_net_inet_ip, IPCTL_ACCEPTSOURCEROUTE, accept_sourceroute, "Enable accepting source routed IP packets"); #define V_ip_acceptsourceroute VNET(ip_acceptsourceroute) -VNET_DEFINE(int, ip_doopts) = 1; /* 0 = ignore, 1 = process, 2 = reject */ +VNET_DEFINE(int, ip_doopts) = 0; /* 0 = ignore, 1 = process, 2 = reject */ SYSCTL_INT(_net_inet_ip, OID_AUTO, process_options, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip_doopts), 0, "Enable IP options processing ([LS]SRR, RR, TS)"); -- 2.11.0 From b5d13b0a2f4365128072e1334099a46f53e8bd49 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 25 Mar 2016 23:25:33 +0100 Subject: [PATCH 191/255] Detach freebsd-version from the build Obtained from: ElectroBSD --- bin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/Makefile b/bin/Makefile index b3385dcd32d9..eb89a529abbe 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -15,7 +15,6 @@ SUBDIR= cat \ echo \ ed \ expr \ - freebsd-version \ getfacl \ hostname \ kenv \ -- 2.11.0 From 19fac2a2f4db5d5fd794ed9941c8823b3566c30f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 25 Mar 2016 18:03:20 +0100 Subject: [PATCH 192/255] geli(8): Dump negative iterations values properly An iterations value of -1 signals that no passphrase is being used. Previously it was dumped incorrectly due to being cast to unsigned first: [fk@elektrobier ~]$ geli dump /dev/ada3p3 Metadata on /dev/ada3p3: magic: GEOM::ELI version: 7 flags: 0x2 ealgo: AES-XTS keylen: 256 provsize: 8589934592 sectorsize: 4096 keys: 0x01 iterations: 4294967295 Salt: [...] Master Key: [...] MD5 hash: 44e17ea9c25f27dff32179bc4f82f659 Obtained from: ElectroBSD --- sys/geom/eli/g_eli.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h index 13e780762549..680f67332dd3 100644 --- a/sys/geom/eli/g_eli.h +++ b/sys/geom/eli/g_eli.h @@ -499,7 +499,7 @@ eli_metadata_dump(const struct g_eli_metadata *md) printf(" provsize: %ju\n", (uintmax_t)md->md_provsize); printf("sectorsize: %u\n", (u_int)md->md_sectorsize); printf(" keys: 0x%02x\n", (u_int)md->md_keys); - printf("iterations: %u\n", (u_int)md->md_iterations); + printf("iterations: %i\n", md->md_iterations); bzero(str, sizeof(str)); for (i = 0; i < sizeof(md->md_salt); i++) { str[i * 2] = hex[md->md_salt[i] >> 4]; -- 2.11.0 From 5a2b5867cfee87c261d87ab508ac5194514355fc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:14:25 +0200 Subject: [PATCH 193/255] geli.8: Document that setkey option -P can't be combined with -i Obtained from: ElectroBSD --- sbin/geom/class/eli/geli.8 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbin/geom/class/eli/geli.8 b/sbin/geom/class/eli/geli.8 index e8a1ef024c1b..3c5111928c9f 100644 --- a/sbin/geom/class/eli/geli.8 +++ b/sbin/geom/class/eli/geli.8 @@ -554,6 +554,8 @@ option. .It Fl P Do not use a passphrase as a component of the new User Key. Cannot be combined with the +.Fl i +or .Fl J option. .El -- 2.11.0 From 5390ab2000076ffe89b760049f8484f7d21c953a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:19:53 +0200 Subject: [PATCH 194/255] geli setkey: Reject combination of -P and -i Iterations aren't used for keys without passphrase and storing a iteration count would result in a passphrase promt that can't be used to attach the provider. Obtained from: ElectroBSD --- sbin/geom/class/eli/geom_eli.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbin/geom/class/eli/geom_eli.c b/sbin/geom/class/eli/geom_eli.c index c804622ddbe2..e8198d118179 100644 --- a/sbin/geom/class/eli/geom_eli.c +++ b/sbin/geom/class/eli/geom_eli.c @@ -1166,6 +1166,8 @@ eli_setkey(struct gctl_req *req) struct g_eli_metadata md; const char *prov; int nargs; + intmax_t iterations; + int nonewpassphrase; nargs = gctl_get_int(req, "nargs"); if (nargs != 1) { @@ -1174,6 +1176,14 @@ eli_setkey(struct gctl_req *req) } prov = gctl_get_ascii(req, "arg0"); + nonewpassphrase = gctl_get_int(req, "nonewpassphrase"); + iterations = gctl_get_intmax(req, "iterations"); + if (iterations != -1 && nonewpassphrase) { + gctl_error(req, + "Options -i and -P are mutually exclusive."); + return; + } + if (eli_metadata_read(req, prov, &md) == -1) return; -- 2.11.0 From a74785b64eff6a45367409611ee999cee746c6fd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:24:02 +0200 Subject: [PATCH 195/255] geli setkey: Prevent passphrase removal if two keys are configured The iterations are shared by both keys and resetting it for one would break the other one as a side-effect. If this is the intention the other key can still be removed with delkey before using setkey for the remaining one. Obtained from: ElectroBSD --- sbin/geom/class/eli/geom_eli.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sbin/geom/class/eli/geom_eli.c b/sbin/geom/class/eli/geom_eli.c index e8198d118179..bd79b34ff1f6 100644 --- a/sbin/geom/class/eli/geom_eli.c +++ b/sbin/geom/class/eli/geom_eli.c @@ -1187,6 +1187,14 @@ eli_setkey(struct gctl_req *req) if (eli_metadata_read(req, prov, &md) == -1) return; + if (nonewpassphrase && bitcount32(md.md_keys) != 1 && + md.md_iterations != -1) { + gctl_error(req, "To be able to switch from passphrase" + "-based key to passphrase-less key, only one can " + "be defined."); + return; + } + if (eli_is_attached(prov)) eli_setkey_attached(req, &md); else -- 2.11.0 From 31c0cc3ae0ad77b15a312ae8fd9c6e2d4fd30011 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:37:03 +0200 Subject: [PATCH 196/255] geli.8: Document that setkey -P can't be used if two user keys with passphrase exist Obtained from: ElectroBSD --- sbin/geom/class/eli/geli.8 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/geom/class/eli/geli.8 b/sbin/geom/class/eli/geli.8 index 3c5111928c9f..22660544db53 100644 --- a/sbin/geom/class/eli/geli.8 +++ b/sbin/geom/class/eli/geli.8 @@ -553,6 +553,9 @@ Cannot be combined with the option. .It Fl P Do not use a passphrase as a component of the new User Key. +This option cannot be used with the +.Cm setkey +subcommand if two User Keys with passphrase are configured. Cannot be combined with the .Fl i or -- 2.11.0 From 0a2332719f7804391b609586f53d00c61380cda2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 14:22:52 +0200 Subject: [PATCH 197/255] geli setkey: Fix key change from passphrase-based to passphrase-less Previously the iterations count wasn't reset resulting in a passphrase prompt that would not accept any passphrase: fk@r500 ~ $sudo geli attach -k /bpool/boot/rpool.key /dev/gpt/rpool-ada1 Enter passphrase: fk@r500 ~ $geli dump /dev/gpt/rpool-ada1 Metadata on /dev/gpt/rpool-ada1: magic: GEOM::ELI version: 7 flags: 0x0 ealgo: AES-XTS keylen: 128 provsize: 1073741824 sectorsize: 512 keys: 0x01 iterations: 463852 Salt: [...] Master Key: [...] MD5 hash: e8693274fecc65d2a13c0071fb9413b3 fk@r500 ~ $sudo geli setkey -K /bpool/boot/rpool.key -P /dev/gpt/rpool-ada1 Note, that the master key encrypted with old keys and/or passphrase may still exists in a metadata backup file. fk@r500 ~ $geli dump /dev/gpt/rpool-ada1 Metadata on /dev/gpt/rpool-ada1: magic: GEOM::ELI version: 7 flags: 0x0 ealgo: AES-XTS keylen: 128 provsize: 1073741824 sectorsize: 512 keys: 0x01 iterations: 463852 Salt: [...] Master Key: [...] MD5 hash: a443402c3b97cb37494283f8f722994d fk@r500 ~ $sudo geli detach gpt/rpool-ada1 fk@r500 ~ $sudo geli attach -k /bpool/boot/rpool.key /dev/gpt/rpool-ada1 Enter passphrase: geli: Wrong key for gpt/rpool-ada1. From userland the promt could be suppressed with "-p" (which is required for passphrase-less keys anyway), but attaching at boot time wasn't possible. PR: 196834 Reported by: Julian Hsiao Obtained from: ElectroBSD --- sbin/geom/class/eli/geom_eli.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sbin/geom/class/eli/geom_eli.c b/sbin/geom/class/eli/geom_eli.c index bd79b34ff1f6..856a862fe0fe 100644 --- a/sbin/geom/class/eli/geom_eli.c +++ b/sbin/geom/class/eli/geom_eli.c @@ -1044,8 +1044,16 @@ eli_setkey_attached(struct gctl_req *req, struct g_eli_metadata *md) /* Check if iterations number should be changed. */ if (val != -1) md->md_iterations = val; - else + else { old = md->md_iterations; + /* + * If the new key does not require a passphrase, + * the iterations count has to be reset to reflect + * this. + */ + if (gctl_get_int(req, "nonewpassphrase") == 1) + md->md_iterations = -1; + } /* Generate key for Master Key encryption. */ if (eli_genkey(req, md, key, true) == NULL) { @@ -1130,7 +1138,14 @@ eli_setkey_detached(struct gctl_req *req, const char *prov, return; } md->md_iterations = val; - } + } else if (gctl_get_int(req, "nonewpassphrase") == 1) { + /* + * If the new key does not require a passphrase, + * the iterations count has to be reset to reflect + * this. + */ + md->md_iterations = -1; + } mkeydst = md->md_mkeys + nkey * G_ELI_MKEYLEN; md->md_keys |= (1 << nkey); -- 2.11.0 From 4a9bea879bc50889c55035ebc3b13a7c97370689 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 Apr 2016 17:37:05 +0200 Subject: [PATCH 198/255] geli.8: Document the fact that both User Keys share an iteration value ... more explicitly. Obtained from: ElectroBSD --- sbin/geom/class/eli/geli.8 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbin/geom/class/eli/geli.8 b/sbin/geom/class/eli/geli.8 index 22660544db53..fdbf60c84240 100644 --- a/sbin/geom/class/eli/geli.8 +++ b/sbin/geom/class/eli/geli.8 @@ -770,6 +770,13 @@ the kernel from the Master Key and cached in memory. The number of Data Keys used by a given provider, and the way they are derived, depend on the GELI version and whether the provider is configured to use data authentication. +.Sh USER KEY LIMITATION +If the first User Key uses a passphrase, the second one has to use +a passphrase as well. +If the first User Key does not use a passphrase, the second one can +not use a passphrase either. +This limitation comes from the metadata format on disk which +currently only stores one iteration count for both keys. .Sh SYSCTL VARIABLES The following .Xr sysctl 8 -- 2.11.0 From 6b7eb1823f801a238e984a845f9609616c97439a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 Apr 2016 11:45:44 +0200 Subject: [PATCH 199/255] etc/rc.d: Remove a couple of requirements that don't apply to ElectroBSD Obtained from: ElectroBSD --- etc/rc.d/NETWORKING | 4 ++-- etc/rc.d/bridge | 2 +- etc/rc.d/ipfw | 1 - etc/rc.d/ipnat | 29 ----------------------------- etc/rc.d/netif | 4 ++-- etc/rc.d/routing | 2 +- etc/rc.d/securelevel | 2 +- 7 files changed, 7 insertions(+), 37 deletions(-) delete mode 100755 etc/rc.d/ipnat diff --git a/etc/rc.d/NETWORKING b/etc/rc.d/NETWORKING index 9cdb5577ed2b..1bbea5aa40b9 100755 --- a/etc/rc.d/NETWORKING +++ b/etc/rc.d/NETWORKING @@ -4,8 +4,8 @@ # # PROVIDE: NETWORKING NETWORK -# REQUIRE: netif netwait netoptions routing ppp ipfw stf -# REQUIRE: defaultroute route6d resolv bridge +# REQUIRE: netif netwait netoptions routing ipfw stf +# REQUIRE: defaultroute routed route6d resolv bridge # REQUIRE: static_arp static_ndp # This is a dummy dependency, for services which require networking diff --git a/etc/rc.d/bridge b/etc/rc.d/bridge index 95e4eb9c2fac..8631fdab2b05 100755 --- a/etc/rc.d/bridge +++ b/etc/rc.d/bridge @@ -26,7 +26,7 @@ # # PROVIDE: bridge -# REQUIRE: netif ppp stf +# REQUIRE: netif stf # KEYWORD: nojail . /etc/rc.subr diff --git a/etc/rc.d/ipfw b/etc/rc.d/ipfw index beccaffaf31b..7f9db1656de2 100755 --- a/etc/rc.d/ipfw +++ b/etc/rc.d/ipfw @@ -4,7 +4,6 @@ # # PROVIDE: ipfw -# REQUIRE: ppp # KEYWORD: nojailvnet . /etc/rc.subr diff --git a/etc/rc.d/ipnat b/etc/rc.d/ipnat deleted file mode 100755 index 6914bceaf466..000000000000 --- a/etc/rc.d/ipnat +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# $FreeBSD$ -# - -# PROVIDE: ipnat -# REQUIRE: ipfilter -# KEYWORD: nojail - -. /etc/rc.subr - -name="ipnat" -desc="user interface to the NAT subsystem" -rcvar="ipnat_enable" -load_rc_config $name -start_cmd="ipnat_start" -stop_cmd="${ipnat_program} -F -C" -reload_cmd="${ipnat_program} -F -C -f ${ipnat_rules}" -extra_commands="reload" -required_files="${ipnat_rules}" -required_modules="ipl:ipfilter" - -ipnat_start() -{ - echo "Installing NAT rules." - ${ipnat_program} -CF -f ${ipnat_rules} ${ipnat_flags} -} - -run_rc_command "$1" diff --git a/etc/rc.d/netif b/etc/rc.d/netif index 6b52c50e7f4f..2458f0a863f5 100755 --- a/etc/rc.d/netif +++ b/etc/rc.d/netif @@ -26,8 +26,8 @@ # # PROVIDE: netif -# REQUIRE: FILESYSTEMS iovctl serial sppp sysctl -# REQUIRE: hostid ipfilter ipfs +# REQUIRE: FILESYSTEMS iovctl serial sysctl +# REQUIRE: hostid # KEYWORD: nojailvnet . /etc/rc.subr diff --git a/etc/rc.d/routing b/etc/rc.d/routing index 2a57fa3b1182..7de71f12c9e1 100755 --- a/etc/rc.d/routing +++ b/etc/rc.d/routing @@ -6,7 +6,7 @@ # # PROVIDE: routing -# REQUIRE: netif ppp stf +# REQUIRE: netif stf # KEYWORD: nojailvnet . /etc/rc.subr diff --git a/etc/rc.d/securelevel b/etc/rc.d/securelevel index c42a03534675..24dbf269df3f 100755 --- a/etc/rc.d/securelevel +++ b/etc/rc.d/securelevel @@ -4,7 +4,7 @@ # # PROVIDE: securelevel -# REQUIRE: adjkerntz ipfw ipfilter pf +# REQUIRE: adjkerntz ipfw pf . /etc/rc.subr -- 2.11.0 From 1d761538e5a3685b017b2c2ca77ffa2d880344e0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 16 Apr 2016 18:17:16 +0200 Subject: [PATCH 200/255] etc/rc.d: Don't try to install ipnat Obtained from: ElectroBSD --- etc/rc.d/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index ac3dda4100a2..213e90c4c8d1 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -46,7 +46,6 @@ FILES= DAEMON \ ipfs \ ipfw \ ipmon \ - ipnat \ ipsec \ ${_kadmind} \ ${_kdc} \ -- 2.11.0 From ff12d15fb3fe883254d3dff457867e11ffe88bdb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 Apr 2016 21:45:39 +0200 Subject: [PATCH 201/255] release/amd64/make-memstick.sh: Use mkimg from the dist directory Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 4263855e011d..3925607dfcd3 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -50,7 +50,8 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} +dist/base/usr/bin/mkimg -y -s gpt -b ${1}/boot/pmbr \ + -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.11.0 From 661b30ec1d2a9cc05337297c5ad945f42b8eb6db Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 18 Mar 2015 13:08:28 +0100 Subject: [PATCH 202/255] parse_mount(): Use vfs.mountroot.timeout for ZFS root pools as well Instead of trying to figure out the required vdevs to wait for, just call kernel_mount() until it works or the time is up. Fancier approaches are conceivable ... If the current approach is kept, it may make sense to add a flag to tell kernel_mount() not to free the mount args. As an alternative the retrying could be delegated to kernel_mount() itself. Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=208882 --- sys/kern/vfs_mountroot.c | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c index 248f6c887c64..ff437c3f4008 100644 --- a/sys/kern/vfs_mountroot.c +++ b/sys/kern/vfs_mountroot.c @@ -707,7 +707,7 @@ parse_mount(char **conf) char *errmsg; struct mntarg *ma; char *dev, *fs, *opts, *tok; - int error; + int delay, error, timeout; error = parse_token(conf, &tok); if (error) @@ -748,15 +748,29 @@ parse_mount(char **conf) if (error != 0) goto out; - ma = NULL; - ma = mount_arg(ma, "fstype", fs, -1); - ma = mount_arg(ma, "fspath", "/", -1); - ma = mount_arg(ma, "from", dev, -1); - ma = mount_arg(ma, "errmsg", errmsg, ERRMSGL); - ma = mount_arg(ma, "ro", NULL, 0); - ma = parse_mountroot_options(ma, opts); - error = kernel_mount(ma, MNT_ROOTFS); + delay = hz / 10; + timeout = root_mount_timeout * hz; + do { + ma = NULL; + ma = mount_arg(ma, "fstype", fs, -1); + ma = mount_arg(ma, "fspath", "/", -1); + ma = mount_arg(ma, "from", dev, -1); + ma = mount_arg(ma, "errmsg", errmsg, ERRMSGL); + ma = mount_arg(ma, "ro", NULL, 0); + ma = parse_mountroot_options(ma, opts); + + error = kernel_mount(ma, MNT_ROOTFS); + if (strcmp(fs, "zfs") != 0) + break; + timeout -= delay; + if (timeout > 0 && error) { + pause("rmdev", delay); + printf("Mounting from %s:%s failed with error %d. " + "%d seconds left. Retrying.\n", fs, dev, error, + timeout / hz); + } + } while (timeout > 0 && error); out: if (error) { printf("Mounting from %s:%s failed with error %d", @@ -960,8 +974,12 @@ vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev) int delay, timeout; /* - * In case of ZFS and NFS we don't have a way to wait for - * specific device. + * For ZFS we can't simply wait for a specific device + * as we only know the pool name. To work around this, + * parse_mount() will retry the mount later on. + * + * While retrying for NFS could be implemented similarly + * it is currently not supported. */ if (strcmp(fs, "zfs") == 0 || strstr(fs, "nfs") != NULL || dev[0] == '\0') { -- 2.11.0 From 8e0d42875aa13a6e8ffd22e401ab95be4aaa6ab5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 1 Nov 2015 14:21:15 +0100 Subject: [PATCH 203/255] parse_mount(): Use the vfs.mountroot.timeout for NFS as well Note that this hasn't actually been tested yet. Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=208882 --- sys/kern/vfs_mountroot.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c index ff437c3f4008..5698ba03d30d 100644 --- a/sys/kern/vfs_mountroot.c +++ b/sys/kern/vfs_mountroot.c @@ -761,7 +761,7 @@ parse_mount(char **conf) ma = parse_mountroot_options(ma, opts); error = kernel_mount(ma, MNT_ROOTFS); - if (strcmp(fs, "zfs") != 0) + if (strcmp(fs, "zfs") != 0 || strstr(fs, "nfs") != NULL) break; timeout -= delay; if (timeout > 0 && error) { @@ -974,12 +974,9 @@ vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev) int delay, timeout; /* - * For ZFS we can't simply wait for a specific device + * For ZFS and NFS we can't simply wait for a specific device * as we only know the pool name. To work around this, * parse_mount() will retry the mount later on. - * - * While retrying for NFS could be implemented similarly - * it is currently not supported. */ if (strcmp(fs, "zfs") == 0 || strstr(fs, "nfs") != NULL || dev[0] == '\0') { -- 2.11.0 From 59bba818e9880c8a728a270ed22cae8e5e4c7de2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 Apr 2016 12:48:33 +0200 Subject: [PATCH 204/255] rc.d: Change geli_autodetach default to 'NO' ... as autodetach leads to panics when scrubbing ZFS pools with more than one device. For details see: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=117158 Obtained from: ElectroBSD --- etc/defaults/rc.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 etc/defaults/rc.conf diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf old mode 100644 new mode 100755 index 18ee897bfb58..c1494f933edf --- a/etc/defaults/rc.conf +++ b/etc/defaults/rc.conf @@ -78,7 +78,7 @@ geli_devices="" # List of devices to automatically attach in addition to geli_tries="" # Number of times to attempt attaching geli device. # If empty, kern.geom.eli.tries will be used. geli_default_flags="" # Default flags for geli(8). -geli_autodetach="YES" # Automatically detach on last close. +geli_autodetach="NO" # Automatically detach on last close. # Providers are marked as such when all file systems are # mounted. # Example use. -- 2.11.0 From 4afd12d729f3e52a22327e70f9be3e5f666e0a71 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 22:05:48 +0200 Subject: [PATCH 205/255] disklatency: Skip invalid io::done probes .. until the cause has been analyzed and fixed Obtained from: ElectroBSD --- share/dtrace/disklatency | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/dtrace/disklatency b/share/dtrace/disklatency index 97ef87a07980..21ff6f719053 100755 --- a/share/dtrace/disklatency +++ b/share/dtrace/disklatency @@ -42,7 +42,7 @@ io:::start } io:::done -/this->start = start_time[arg0]/ +/(this->start = start_time[arg0]) && (args[1]->unit_number != -1)/ { this->delta = (timestamp - this->start) / 1000; @q[args[1]->device_name, args[1]->unit_number] = -- 2.11.0 From 8ea52afede3ab317c11bad8a8d7a61d121f2b1e0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 22:06:51 +0200 Subject: [PATCH 206/255] disklatency: Reformat output to print device name and unit together While at it, add a delimiter. Obtained from: ElectroBSD --- share/dtrace/disklatency | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/share/dtrace/disklatency b/share/dtrace/disklatency index 21ff6f719053..930be21d0afb 100755 --- a/share/dtrace/disklatency +++ b/share/dtrace/disklatency @@ -55,10 +55,11 @@ io:::done tick-10s { - printa(" %s (%d), us:\n%@d\n", @q); - printa("max%s (%d), us:\n%@d\n", @max); - printa("avg%s (%d), us:\n%@d\n", @avg); - printa("stddev%s (%d), us:\n%@d\n", @stddev); + printf("---------------------------------------------\n"); + printa("%s%d, us:\n%@d\n", @q); + printa("%s%d max, us:\n%@d\n", @max); + printa("%s%d avg, us:\n%@d\n", @avg); + printa("%s%d stdev, us:\n%@d\n", @stddev); clear(@q); clear(@max); -- 2.11.0 From 0692eb5a428c73727d4ccee7e4d293ea64693f9b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 15 May 2016 09:30:31 +0200 Subject: [PATCH 207/255] disklatency: Use trunc() instead of clear() ... so only latency stats for disks with activity in the relevant intervals are shown. Obtained from: ElectroBSD --- share/dtrace/disklatency | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/share/dtrace/disklatency b/share/dtrace/disklatency index 930be21d0afb..f4081c14bc84 100755 --- a/share/dtrace/disklatency +++ b/share/dtrace/disklatency @@ -61,8 +61,8 @@ tick-10s printa("%s%d avg, us:\n%@d\n", @avg); printa("%s%d stdev, us:\n%@d\n", @stddev); - clear(@q); - clear(@max); - clear(@avg); - clear(@stddev); + trunc(@q); + trunc(@max); + trunc(@avg); + trunc(@stddev); } -- 2.11.0 From 1c5d462c25956b56000674a67cae44747c8b74db Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 1 May 2016 16:27:55 +0200 Subject: [PATCH 208/255] vmstat: Prevent output truncation when piping zone statistics ... into other programs. Apparently libxo (currently?) can't be trusted to flush its internal buffer before it's full, triggering the flushing manually works around this. Before (sometimes): fk@r500 ~ $vmstat -z | wc -c 8192 After (consistently): fk@r500 ~ $vmstat -z | wc -c 24156 Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=206128 --- usr.bin/vmstat/vmstat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c index 0dbd2d165dda..a4d6f25786c1 100644 --- a/usr.bin/vmstat/vmstat.c +++ b/usr.bin/vmstat/vmstat.c @@ -1536,6 +1536,7 @@ domemstat_zone(void) memstat_get_numallocs(mtp), memstat_get_failures(mtp), memstat_get_sleeps(mtp)); xo_close_instance("zone"); + xo_flush(); } memstat_mtl_free(mtlp); xo_close_list("zone"); -- 2.11.0 From 43062beb3daa40e0fa8ceee0d79e936f9dfdcacd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 May 2016 14:48:21 +0200 Subject: [PATCH 209/255] gptzfsboot: Print Elektrobier after failing to locate a ZFS pool Obtained from: ElectroBSD --- sys/boot/i386/zfsboot/zfsboot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/i386/zfsboot/zfsboot.c b/sys/boot/i386/zfsboot/zfsboot.c index 399851152a2b..62332d83cd4c 100644 --- a/sys/boot/i386/zfsboot/zfsboot.c +++ b/sys/boot/i386/zfsboot/zfsboot.c @@ -706,7 +706,7 @@ main(void) if (!spa) { printf("%s: No ZFS pools located, can't boot\n", BOOTPROG); for (;;) - ; + printf("Elektrobier, "); } primary_spa = spa; -- 2.11.0 From ee6afc737af36e5a8f2c79cba954e64dbbcee7eb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 May 2016 16:02:08 +0200 Subject: [PATCH 210/255] uma: Add UMA_ZONE_NODRAIN flag INCOMPLETE and non-functional Obtained from: ElectroBSD --- sys/vm/uma.h | 2 +- sys/vm/uma_core.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sys/vm/uma.h b/sys/vm/uma.h index f4c2de8a5d25..fcdb7449f830 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -262,7 +262,7 @@ uma_zone_t uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, * information in the vm_page. */ #define UMA_ZONE_SECONDARY 0x0200 /* Zone is a Secondary Zone */ -/* 0x0400 Unused */ +#define UMA_ZONE_NODRAIN 0x0400 /* Do not drain zone while in use */ #define UMA_ZONE_MAXBUCKET 0x0800 /* Use largest buckets */ #define UMA_ZONE_CACHESPREAD 0x1000 /* * Spread memory start locations across diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index e54db2a29740..560851bb10e8 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -855,6 +855,9 @@ keg_drain(uma_keg_t keg) if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) return; + if (keg->uk_flags & UMA_ZONE_NODRAIN) + return; + #ifdef UMA_DEBUG printf("%s free items: %u\n", keg->uk_name, keg->uk_free); #endif @@ -2077,6 +2080,7 @@ uma_zdestroy(uma_zone_t zone) { sx_slock(&uma_drain_lock); + zone->uz_flags &= ~UMA_ZONE_NODRAIN; zone_free_item(zones, zone, NULL, SKIP_NONE); sx_sunlock(&uma_drain_lock); } -- 2.11.0 From e99e43a06cafe195340da3eae322b9271fb264d4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 May 2016 13:06:34 +0200 Subject: [PATCH 211/255] zone.9: Stop documenting UMA_ZONE_REFCNT which was removed in r296243 Obtained from: ElectroBSD --- share/man/man9/zone.9 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/share/man/man9/zone.9 b/share/man/man9/zone.9 index 69d1f20b498e..798be1c20fdf 100644 --- a/share/man/man9/zone.9 +++ b/share/man/man9/zone.9 @@ -155,10 +155,6 @@ is a subset of the following flags: .Bl -tag -width "foo" .It Dv UMA_ZONE_NOFREE Slabs of the zone are never returned back to VM. -.It Dv UMA_ZONE_REFCNT -Each item in the zone would have internal reference counter associated with it. -See -.Fn uma_find_refcnt . .It Dv UMA_ZONE_NODUMP Pages belonging to the zone will not be included into mini-dumps. .It Dv UMA_ZONE_PCPU -- 2.11.0 From 385c138caf01687465814a8a8cc7a361eb94bd47 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 16:53:50 +0200 Subject: [PATCH 212/255] uma.h: Rename uma_prealloc()'s second argument to nitems to increase consistency Obtained from: ElectroBSD --- sys/vm/uma.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/vm/uma.h b/sys/vm/uma.h index fcdb7449f830..b1fc43f713cf 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -613,14 +613,14 @@ void uma_zone_set_freef(uma_zone_t zone, uma_free freef); * * Arguments: * zone The zone to fill - * itemcnt The number of items to reserve + * nitems The number of items to reserve * * Returns: * Nothing * * NOTE: This is blocking and should only be done at startup */ -void uma_prealloc(uma_zone_t zone, int itemcnt); +void uma_prealloc(uma_zone_t zone, int nitems); /* * Used to determine if a fixed-size zone is exhausted. -- 2.11.0 From cfe0f2ef947b4f9071763332ee9a6a2e7a0e093f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 17:25:00 +0200 Subject: [PATCH 213/255] zone.9: Document uma_prealloc() and uma_zone_reserve() Obtained from: ElectroBSD --- share/man/man9/zone.9 | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/share/man/man9/zone.9 b/share/man/man9/zone.9 index 798be1c20fdf..3636a5ecfa98 100644 --- a/share/man/man9/zone.9 +++ b/share/man/man9/zone.9 @@ -29,6 +29,7 @@ .Dt ZONE 9 .Os .Sh NAME +.Nm uma_prealloc , .Nm uma_zcreate , .Nm uma_zalloc , .Nm uma_zalloc_arg , @@ -36,6 +37,7 @@ .Nm uma_zfree_arg , .Nm uma_find_refcnt , .Nm uma_zdestroy , +.Nm uma_zone_reserve , .Nm uma_zone_set_max, .Nm uma_zone_get_max, .Nm uma_zone_get_cur, @@ -46,6 +48,8 @@ .In sys/param.h .In sys/queue.h .In vm/uma.h +.Ft void +.Fn uma_prealloc "uma_zone_t zone" "int nitems" .Ft uma_zone_t .Fo uma_zcreate .Fa "char *name" "int size" @@ -64,6 +68,8 @@ .Fn uma_find_refcnt "uma_zone_t zone" "void *item" .Ft void .Fn uma_zdestroy "uma_zone_t zone" +.Ft void +.Fn uma_zone_reserve "uma_zone_t zone" "int nitems" .Ft int .Fn uma_zone_set_max "uma_zone_t zone" "int nitems" .Ft int @@ -272,6 +278,21 @@ must have been freed with before. .Pp The +.Fn uma_prealloc +function pre-fills a zone with items. +This function is blocking and should be called before the zone is used. +.Pp +The +.Fn uma_zone_reserve +function sets a reserve of items to hold for +.Dv M_USE_RESERVE +allocations. +No new items are allocated. +The +.Fn uma_prealloc +function can be used to fill the reserve before it is needed. +.Pp +The .Fn uma_zone_set_max function limits the number of items .Pq and therefore memory -- 2.11.0 From 0d9d7a18d7dfe526f86db413a2de85a4202b99cd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 16 May 2016 12:28:50 +0200 Subject: [PATCH 214/255] uma.h: Stop claiming that uma_zcreate() may return NULL ... 'if the wait flag is not set'. The function does not actually accept a "wait flag". Internally it unconditionally uses the M_WAITOK flag when calling zone_alloc_item(). Obtained from: ElectroBSD --- sys/vm/uma.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/vm/uma.h b/sys/vm/uma.h index b1fc43f713cf..ce20edc82004 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -173,7 +173,7 @@ typedef void (*uma_release)(void *arg, void **store, int count); * * Returns: * A pointer to a structure which is intended to be opaque to users of - * the interface. The value may be null if the wait flag is not set. + * the interface. */ uma_zone_t uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, uma_fini fini, -- 2.11.0 From 55646694134799761db0b297e3177f2b62a32ddc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 22 May 2016 12:47:29 +0200 Subject: [PATCH 215/255] ipfw: Prefill the dynamic rule zone and prevent uma from freeing unused items Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=209680 --- sys/netpfil/ipfw/ip_fw_dynamic.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sys/netpfil/ipfw/ip_fw_dynamic.c b/sys/netpfil/ipfw/ip_fw_dynamic.c index 7dd72e47dd9e..f1fda48d28a6 100644 --- a/sys/netpfil/ipfw/ip_fw_dynamic.c +++ b/sys/netpfil/ipfw/ip_fw_dynamic.c @@ -1399,11 +1399,14 @@ ipfw_dyn_init(struct ip_fw_chain *chain) V_ipfw_dyn_rule_zone = uma_zcreate("IPFW dynamic rule", sizeof(ipfw_dyn_rule), NULL, NULL, NULL, NULL, - UMA_ALIGN_PTR, 0); + UMA_ALIGN_PTR, UMA_ZONE_NOFREE); /* Enforce limit on dynamic rules */ uma_zone_set_max(V_ipfw_dyn_rule_zone, V_dyn_max); + /* Prefill the zone to make sure the limit is always reachable */ + uma_prealloc(V_ipfw_dyn_rule_zone, V_dyn_max); + callout_init(&V_ipfw_timeout, 1); /* -- 2.11.0 From bd12ddf34f2a3447991108428e7d255b1c078ccd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 13:15:31 +0200 Subject: [PATCH 216/255] (Hopefully) make TCP/IP connections reliable under memory pressure again ... by bringing back the UMA_ZONE_NOFREE flag for a couple of TCP related UMA zones. This reverts r297740/f3240321 completely and partially reverts r297738/55f63f4b6bf and r297742/0a1a3e73fc. Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=209680 --- sys/netinet/tcp_fastopen.c | 2 +- sys/netinet/tcp_subr.c | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sys/netinet/tcp_fastopen.c b/sys/netinet/tcp_fastopen.c index e2b3b8637f60..730032a63daf 100644 --- a/sys/netinet/tcp_fastopen.c +++ b/sys/netinet/tcp_fastopen.c @@ -205,7 +205,7 @@ void tcp_fastopen_init(void) { V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int), - NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); rm_init(&V_tcp_fastopen_keylock, "tfo_keylock"); callout_init_rm(&V_tcp_fastopen_autokey_ctx.c, &V_tcp_fastopen_keylock, 0); diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index 662c435a0e72..1c45a9a6a117 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -651,13 +651,14 @@ tcp_init(void) hashsize); } in_pcbinfo_init(&V_tcbinfo, "tcp", &V_tcb, hashsize, hashsize, - "tcp_inpcb", tcp_inpcb_init, NULL, 0, IPI_HASHFIELDS_4TUPLE); + "tcp_inpcb", tcp_inpcb_init, NULL, UMA_ZONE_NOFREE, + IPI_HASHFIELDS_4TUPLE); /* * These have to be type stable for the benefit of the timers. */ V_tcpcb_zone = uma_zcreate("tcpcb", sizeof(struct tcpcb_mem), - NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); uma_zone_set_max(V_tcpcb_zone, maxsockets); uma_zone_set_warning(V_tcpcb_zone, "kern.ipc.maxsockets limit reached"); @@ -667,7 +668,7 @@ tcp_init(void) TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack); V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole), - NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); /* Skip initialization of globals for non-default instances. */ if (!IS_DEFAULT_VNET(curvnet)) -- 2.11.0 From 76e266d259a448abdf65e7e8a65b20db2ffce015 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 15 May 2016 12:09:26 +0200 Subject: [PATCH 217/255] Add geli-request-monitor.d ... which shows the BIO commands GELI received and their lengths from the caller's point of view. Obtained from: ElectroBSD --- share/dtrace/geli-request-monitor.d | 86 +++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 share/dtrace/geli-request-monitor.d diff --git a/share/dtrace/geli-request-monitor.d b/share/dtrace/geli-request-monitor.d new file mode 100755 index 000000000000..c306d15bff96 --- /dev/null +++ b/share/dtrace/geli-request-monitor.d @@ -0,0 +1,86 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * geli-request-monitor + * + * Shows the length distribution of various GELI commands from + * the callers point of view (that is, without geli overhead). + * + * Copyright (c) 2016 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet + +dtrace:::BEGIN +{ + bio_command[0x01] = "BIO_READ"; + bio_command[0x02] = "BIO_WRITE"; + bio_command[0x04] = "BIO_DELETE"; + bio_command[0x08] = "BIO_GETATTR"; + bio_command[0x10] = "BIO_FLUSH"; + bio_command[0x20] = "BIO_CMD0"; + bio_command[0x40] = "BIO_CMD1"; + bio_command[0x80] = "BIO_CMD2"; +} + + +fbt::g_eli_crypto_run:entry +{ + this->bp = (struct bio *)arg1; + this->sc = (struct g_eli_softc *)this->bp->bio_to->geom->softc; + + this->geom_name = stringof(this->sc->sc_geom->name); + this->length = this->bp->bio_length; + this->inflight = this->sc->sc_inflight; + + this->bio_command = (bio_command[this->bp->bio_cmd] != NULL) ? + bio_command[this->bp->bio_cmd] : "Unregistered command"; + + @inflight_max[this->geom_name] = max(this->inflight); + @inflight_max_total[this->geom_name] = max(this->inflight); + + @length[this->geom_name, this->bio_command] = + lquantize(this->length, 4096, 172032, 4096); + @length_total[this->geom_name, this->bio_command] = + lquantize(this->length, 4096, 172032, 4096); +/* + printf("%s: Length: %d, Inflight: %d", this->bio_command, + this->length, this->inflight); +*/ + + stats_available = 1; +} + +tick-60s +/stats_available/ +{ + printf("------------------------------\n"); + printa(@inflight_max); + printa(@length); + trunc(@length); + trunc(@inflight_max); + stats_available = 0; +} + +END +{ + /* Explicitly ditch interval stats so they aren't printed */ + trunc(@length); + trunc(@inflight_max); + + printf("\n------------------------------\n"); + printa(@inflight_max_total); + printa(@length_total); +} -- 2.11.0 From 65c5f19321d70c40c87cef5bb5b8741ac32d013a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 12 May 2016 14:52:16 +0200 Subject: [PATCH 218/255] GELI: Use a dedicated uma zone for writes to onetime devices ... as they are likely to originate from the vm page daemon. Previously the system could deadlock because the vm daemon was waiting for pages to be written to disk, while GELI was waiting for the vm daemon to make room for the buffer GELI needed to actually write the pages: (kgdb) where #0 sched_switch (td=0xfffff800055bf9a0, newtd=0xfffff80002341000, flags=) at /usr/src/sys/kern/sched_ule.c:1969 #1 0xffffffff80962635 in mi_switch (flags=, newtd=0x0) at /usr/src/sys/kern/kern_synch.c:455 #2 0xffffffff809aaa3a in sleepq_wait (wchan=0x0, pri=0) at /usr/src/sys/kern/subr_sleepqueue.c:637 #3 0xffffffff80962038 in _sleep (ident=, lock=, priority=, wmesg=0xffffffff80e826ee "vmwait", sbt=0, pr=, flags=) at /usr/src/sys/kern/kern_synch.c:229 #4 0xffffffff80c1ac6b in vm_wait () at /usr/src/sys/vm/vm_page.c:2705 #5 0xffffffff80c06a9f in kmem_back (object=0xffffffff8144d6f0, addr=18446741874805047296, size=69632, flags=) at /usr/src/sys/vm/vm_kern.c:356 #6 0xffffffff80c068d2 in kmem_malloc (vmem=0xffffffff813aa500, size=69632, flags=2) at /usr/src/sys/vm/vm_kern.c:316 #7 0xffffffff80bfd7d6 in uma_large_malloc (size=69632, wait=2) at /usr/src/sys/vm/uma_core.c:1106 #8 0xffffffff8092f614 in malloc (size=, mtp=0xffffffff81b4d520, flags=0) at /usr/src/sys/kern/kern_malloc.c:513 #9 0xffffffff81b4ab99 in g_eli_crypto_run (wr=0xfffff80002560040, bp=0xfffff80008a86d90) at /usr/src/sys/modules/geom/geom_eli/../../../geom/eli/g_eli_privacy.c:262 #10 0xffffffff81b3e860 in g_eli_worker (arg=0xfffff80002560040) at /usr/src/sys/modules/geom/geom_eli/../../../geom/eli/g_eli.c:565 #11 0xffffffff80910f5c in fork_exit (callout=0xffffffff81b3e0b0 , arg=0xfffff80002560040, frame=0xfffffe005005ec00) at /usr/src/sys/kern/kern_fork.c:1034 #12 0xffffffff80c33f0e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:611 #13 0x0000000000000000 in ?? () (kgdb) p vm_cnt $16 = {v_swtch = 0, v_trap = 0, v_syscall = 0, v_intr = 0, v_soft = 0, v_vm_faults = 0, v_io_faults = 0, v_cow_faults = 0, v_cow_optim = 0, v_zfod = 0, v_ozfod = 0, v_swapin = 0, v_swapout = 0, v_swappgsin = 0, v_swappgsout = 0, v_vnodein = 0, v_vnodeout = 0, v_vnodepgsin = 0, v_vnodepgsout = 0, v_intrans = 0, v_reactivated = 0, v_pdwakeups = 22197, v_pdpages = 0, v_tcached = 0, v_dfree = 0, v_pfree = 0, v_tfree = 0, v_page_size = 4096, v_page_count = 247688, v_free_reserved = 372, v_free_target = 5320, v_free_min = 1609, v_free_count = 2, v_wire_count = 140735, v_active_count = 96194, v_inactive_target = 7980, v_inactive_count = 10756, v_cache_count = 0, v_pageout_free_min = 34, v_interrupt_free_min = 2, v_free_severe = 990, v_forks = 0, v_vforks = 0, v_rforks = 0, v_kthreads = 0, v_forkpages = 0, v_vforkpages = 0, v_rforkpages = 0, v_kthreadpages = 0, v_spare = 0xffffffff8144d5ac} A sysctl is added to optionally use the zone for GELI writes in general, without letting common writes cut into the reserve for onetime writes. This may reduce latency for larger writes and as we need to keep a couple of items in the zone anyway, the impact on the zone size is minor. Initial testing seems to indicate that the sysctl could be safely enabled by default in the future. Currently a single zone with a somewhat humongous item size sufficient for all GELI writes is being used. While this may look a bit wasteful, in practice we don't need a lot of items, so this seem tolerable for now. The best solution would probably be to only use the dedicated uma zone for common writes if the size is above 65356 bytes, the largest zone item size internally used by malloc. Currently the zone isn't used for reads as those are less time critical and usually are small enough for malloc() to succeed right away anyway. Example length distribution when reproducing ElectroBSD with -j4 and 1 GB of RAM: gpt/swap-ada1.eli BIO_WRITE value ------------- Distribution ------------- count < 4096 | 0 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 4965848 8192 |@@@@@ 943980 12288 |@@ 362668 16384 |@ 161485 20480 |@ 120939 24576 | 87827 28672 | 57402 32768 | 40470 36864 | 42243 40960 | 28543 45056 | 20347 49152 | 15235 53248 | 13450 57344 | 9535 61440 | 9952 65536 |@ 179360 69632 | 0 gpt/swap-ada1.eli BIO_READ value ------------- Distribution ------------- count < 4096 | 0 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 4645114 8192 | 0 12288 | 0 16384 | 3446 20480 | 0 Note that the GELI overhead is not accounted for here and only the results for the swap device are shown. Zone use: [fk@elektrobier3 ~]$ vmstat -z | egrep 'ITEM|eli' | column -t ITEM SIZE LIMIT USED FREE REQ FAIL SLEEP g_eli: 172032, 0, 0, 14, 8077487, 0, 0 This includes writes to gpt/dpool-ada1.eli and gpt/dpool-ada1.eli. Discussion: While the zone served 8077487 memory requests total, 14 items were sufficient for this and therefore the zone only withheld 172032 * 14 bytes plus zone meta data from the rest of the system. Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 26 ++++++++++++++++++++++++-- sys/geom/eli/g_eli.h | 18 ++++++++++++++++++ sys/geom/eli/g_eli_privacy.c | 35 +++++++++++++++++++++++++++++++++-- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 9add13044a6d..4c6309c9af7e 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -82,6 +82,17 @@ u_int g_eli_batch = 0; SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0, "Use crypto operations batching"); +uma_zone_t g_eli_zone; +static u_int g_eli_uma_reserve = 1; +SYSCTL_UINT(_kern_geom_eli, OID_AUTO, uma_reserve, CTLFLAG_RDTUN, + &g_eli_uma_reserve, 0, "Items to pre-allocate in dedicated uma zone " + "and reserve for writes to onetime disks"); + +u_int g_eli_all_writes_use_uma = 0; +SYSCTL_UINT(_kern_geom_eli, OID_AUTO, use_uma_for_all_writes, CTLFLAG_RDTUN, + &g_eli_all_writes_use_uma, 0, "Use the dedicated uma zone for all writes. " + "May reduce write latency but also inflates memory use a bit"); + /* * Passphrase cached during boot, in order to be more user-friendly if * there are multiple providers using the same passphrase. @@ -246,7 +257,12 @@ g_eli_write_done(struct bio *bp) pbp->bio_inbed++; if (pbp->bio_inbed < pbp->bio_children) return; - free(pbp->bio_driver2, M_ELI); + sc = pbp->bio_to->geom->softc; + if (g_eli_all_writes_use_uma || + (sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) + uma_zfree(g_eli_zone, pbp->bio_driver2); + else + free(pbp->bio_driver2, M_ELI); pbp->bio_driver2 = NULL; if (pbp->bio_error != 0) { G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__, @@ -258,7 +274,6 @@ g_eli_write_done(struct bio *bp) /* * Write is finished, send it up. */ - sc = pbp->bio_to->geom->softc; g_io_deliver(pbp, pbp->bio_error); if (sc != NULL) atomic_subtract_int(&sc->sc_inflight, 1); @@ -1254,6 +1269,12 @@ static void g_eli_init(struct g_class *mp) { + g_eli_zone = uma_zcreate("g_eli", ELI_ZONE_ITEM_SIZE, NULL, NULL, + NULL, NULL, 0, UMA_ZONE_NOFREE); + /* Increase the chances that items are available when needed. */ + uma_prealloc(g_eli_zone, g_eli_uma_reserve); + uma_zone_reserve(g_eli_zone, g_eli_uma_reserve); + g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync, g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST); if (g_eli_pre_sync == NULL) @@ -1264,6 +1285,7 @@ static void g_eli_fini(struct g_class *mp) { + uma_zdestroy(g_eli_zone); if (g_eli_pre_sync != NULL) EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync); } diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h index 680f67332dd3..4119348f68a6 100644 --- a/sys/geom/eli/g_eli.h +++ b/sys/geom/eli/g_eli.h @@ -139,6 +139,24 @@ #define G_ELI_CRYPTO_SW 2 #ifdef _KERNEL +/* + * Items in the dedicated uma zone have a fixed size and need + * to be big enough for all write lengths. + * + * MAXPHYS is the largest amount of data geli can receive in a row, + * additionally we have to account for the encryption overhead, which + * depends on the number of sectors. + * + * 512 bytes is the smallest sector size supported and results in the + * largest overhead. If larger sectors are being used, we'll just waste + * a bit more memory. + * + * Given that the zone does not need a lot of items, the generous + * item size seems tolerable for now. + */ +#define ELI_ZONE_ITEM_SIZE (MAXPHYS + (MAXPHYS / 512) * \ + (sizeof(struct cryptop) + sizeof(struct cryptodesc))) + extern int g_eli_debug; extern u_int g_eli_overwrites; extern u_int g_eli_batch; diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c index 6ed584684092..282b88627597 100644 --- a/sys/geom/eli/g_eli_privacy.c +++ b/sys/geom/eli/g_eli_privacy.c @@ -49,6 +49,9 @@ __FBSDID("$FreeBSD$"); #include #include +extern u_int g_eli_all_writes_use_uma; +extern uma_zone_t g_eli_zone; + /* * Code paths: * BIO_READ: @@ -153,7 +156,11 @@ g_eli_crypto_write_done(struct cryptop *crp) if (bp->bio_error != 0) { G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", bp->bio_error); - free(bp->bio_driver2, M_ELI); + if (g_eli_all_writes_use_uma || + (sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) + uma_zfree(g_eli_zone, bp->bio_driver2); + else + free(bp->bio_driver2, M_ELI); bp->bio_driver2 = NULL; g_destroy_bio(cbp); g_io_deliver(bp, bp->bio_error); @@ -259,8 +266,32 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) */ if (bp->bio_cmd == BIO_WRITE) size += bp->bio_length; - p = malloc(size, M_ELI, M_WAITOK); + if (bp->bio_cmd == BIO_WRITE && + (((sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) || + g_eli_all_writes_use_uma)) { + int uma_flags; + + KASSERT(size <= ELI_ZONE_ITEM_SIZE, + ("Insufficient ELI_ZONE_ITEM_SIZE %u < %u", + (unsigned)ELI_ZONE_ITEM_SIZE, (unsigned)size)); + /* + * Writes to onetime providers are likely to originate + * from the page daemon, therefore we try to get the + * memory a bit harder for them to prevent vm deadlocks. + */ + if ((sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) + uma_flags = M_NOWAIT|M_USE_RESERVE; + else + uma_flags = M_WAITOK; + + while (NULL == (p = uma_zalloc(g_eli_zone, uma_flags))) { + /* Only reachable for onetime providers */ + pause("g_eli:uma", min(hz/1000, 1)); + } + } else { + p = malloc(size, M_ELI, M_WAITOK); + } bp->bio_inbed = 0; bp->bio_children = nsec; bp->bio_driver2 = p; -- 2.11.0 From dc6cb10504b03d32b035a4b1112b7c4da5b196de Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 27 May 2016 18:55:19 +0200 Subject: [PATCH 219/255] atkbdc: Work around (apparently) broken mouse reinitialization ... by increasing pkterrthresh to 10. Previously moving the mouse while loading DTrace modules, for example by running a DTrace script without having loaded the modules manually first, would reliably disables the mouse (track point) until reboot (on a ThinkPad R500). With debug.psm.loglevel=5 the logs showed: Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 03 fe 01 ff 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 02 ff 01 ff 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 03 fe 02 ff 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 08 02 00 03 fe 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 01 ff 03 fe 00 Nov 25 13:38:07 r500 kernel: [132] psmintr: 18 fe 00 03 ff 00 Nov 25 13:38:08 r500 sudo: fk : TTY=ttyv0 ; PWD=/home/fk ; USER=root ; COMMAND=/usr/share/dtrace/toolkit/execsnoop Nov 25 13:38:08 r500 kernel: [134] psm0: lost interrupt? Nov 25 13:38:09 r500 kernel: [134] psmintr: 08 00 01 03 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 08 00 01 04 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 01 04 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 02 05 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 02 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fe 03 05 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fe 04 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fd 05 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fd 05 03 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fe 03 05 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fd 03 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fa 06 05 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fc 04 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fa 03 04 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f9 05 07 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f7 03 05 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f9 04 06 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f8 02 04 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fb 01 04 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fb 01 02 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 00 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 18 ff 00 02 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: c8 d3 0b 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: out of sync (00c0 != 0000) 881 cmds since last error. Nov 25 13:38:11 r500 kernel: [136] psmintr: discard a byte (1) Nov 25 13:38:11 r500 kernel: [136] psmintr: d3 0b 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: out of sync (00c0 != 0000) 0 cmds since last error. Nov 25 13:38:11 r500 kernel: [136] psmintr: discard a byte (2) Nov 25 13:38:11 r500 kernel: [136] psmintr: 0b 08 4b 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 3c 08 0c 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0f 08 0c 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 10 08 0b 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 10 08 0a 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 12 08 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 10 08 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 12 08 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 13 08 05 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0f 08 05 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 13 08 04 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 13 08 01 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0b 08 00 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0a 18 ff 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0f 18 fd 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0e 18 fc 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 08 18 fc 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0a 18 fc 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 fb 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 fa 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 f5 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 09 18 f2 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 08 18 f2 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 f0 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 06 18 f2 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 03 18 ee 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 02 18 f3 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 01 18 ef 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 00 38 ed 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: ff 38 ee 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: out of sync (00c0 != 0000) 0 cmds since last error. Nov 25 13:38:11 r500 kernel: [136] psmintr: reset the mouse. Nov 25 13:38:12 r500 kernel: [137] psm0: current command byte: 0047 (reinitialize). Nov 25 13:38:12 r500 kernel: [137] psm: DISABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: ENABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: DISABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_SAMPLING_RATE (100) 00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_RESOLUTION (2) 00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_SCALING11 return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_STREAM_MODE return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SEND_AUX_DEV_STATUS return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: status 00 02 64 Nov 25 13:38:12 r500 kernel: [137] psm: ENABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SEND_AUX_DEV_STATUS return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: status 20 02 64 Nov 25 13:38:15 r500 kernel: [140] psm0: lost interrupt? Nov 25 13:38:16 r500 kernel: [141] psm0: lost interrupt? Nov 25 13:38:17 r500 kernel: [142] psm0: lost interrupt? Nov 25 13:38:18 r500 kernel: [143] psm0: lost interrupt? Nov 25 13:38:19 r500 kernel: [144] psm0: lost interrupt? Nov 25 13:38:20 r500 kernel: [145] psm0: lost interrupt? Nov 25 13:38:21 r500 kernel: [147] psm0: lost interrupt? Nov 25 13:38:22 r500 kernel: [148] psm0: lost interrupt? Nov 25 13:38:23 r500 kernel: [149] psm0: lost interrupt? Nov 25 13:38:24 r500 kernel: [150] psm0: lost interrupt? Nov 25 13:38:25 r500 kernel: [151] psm0: lost interrupt? Nov 25 13:38:26 r500 kernel: [152] psm0: lost interrupt? Nov 25 13:38:27 r500 kernel: [153] psm0: lost interrupt? Nov 25 13:38:29 r500 kernel: [154] psm0: lost interrupt? Nov 25 13:38:30 r500 kernel: [155] psm0: lost interrupt? Nov 25 13:38:31 r500 kernel: [156] psm0: lost interrupt? Nov 25 13:38:32 r500 kernel: [157] psm0: lost interrupt? Nov 25 13:38:33 r500 kernel: [158] psm0: lost interrupt? Nov 25 13:38:34 r500 kernel: [159] psm0: lost interrupt? Nov 25 13:38:35 r500 kernel: [160] psm0: lost interrupt? Nov 25 13:38:36 r500 kernel: [161] psm0: lost interrupt? Nov 25 13:38:37 r500 kernel: [162] psm0: lost interrupt? Nov 25 13:38:38 r500 kernel: [163] psm0: lost interrupt? Nov 25 13:38:39 r500 kernel: [164] psm0: lost interrupt? Nov 25 13:38:40 r500 kernel: [165] psm0: lost interrupt? After the "reset" the mouse cursor no longer moved and rebooting seemed to be the only "cure". Setting debug.psm.pkterrthresh=10 seems to work around the issue, so does not moving the cursor until the modules are loaded. Mouse resets for other reasons seemed to occasionally render the mouse useless as well, but triggering the problem by loading DTrace modules is the only reproducible method currently known. Obtained from: ElectroBSD --- sys/dev/atkbdc/psm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/atkbdc/psm.c b/sys/dev/atkbdc/psm.c index e5f9838c91ba..234d55806a71 100644 --- a/sys/dev/atkbdc/psm.c +++ b/sys/dev/atkbdc/psm.c @@ -2457,7 +2457,7 @@ SYSCTL_INT(_debug_psm, OID_AUTO, secs, CTLFLAG_RW, &psmsecs, 0, static int psmusecs = 500000; SYSCTL_INT(_debug_psm, OID_AUTO, usecs, CTLFLAG_RW, &psmusecs, 0, "Microseconds to add to psmsecs"); -static int pkterrthresh = 2; +static int pkterrthresh = 10; SYSCTL_INT(_debug_psm, OID_AUTO, pkterrthresh, CTLFLAG_RW, &pkterrthresh, 0, "Number of error packets allowed before reinitializing the mouse"); -- 2.11.0 From 6bcaa79252e27091d1fb7e8ae92c8da34555c9b5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 16:36:46 +0200 Subject: [PATCH 220/255] vm_pageout_scan(): Add SDT probes to make a couple of internal variables visible Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 67cf415d2e1e..3e36f7c1c3fa 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -140,6 +140,15 @@ SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, SDT_PROVIDER_DEFINE(vm); SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan); +SDT_PROBE_DEFINE4(vm, , , before__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int deficit"); +SDT_PROBE_DEFINE5(vm, , , after__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int addl_page_shortage", + "int vnodes_skipped"); +SDT_PROBE_DEFINE3(vm, , , before__active__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage"); +SDT_PROBE_DEFINE3(vm, , , after__active__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage"); #if !defined(NO_SWAPPING) /* the kernel process "vm_daemon"*/ @@ -921,6 +930,9 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) vnodes_skipped = 0; + SDT_PROBE4(vm, , , before__inactive__scan, vmd, pass, page_shortage, + deficit); + /* * Start scanning the inactive queue for pages that we can free. The * scan will stop when we reach the target or we have scanned the @@ -1146,6 +1158,9 @@ relock_queue: } vm_pagequeue_unlock(pq); + SDT_PROBE5(vm, , , after__inactive__scan, vmd, pass, page_shortage, + addl_page_shortage, vnodes_skipped); + #if !defined(NO_SWAPPING) /* * Wakeup the swapout daemon if we didn't free the targeted number of @@ -1194,6 +1209,8 @@ relock_queue: if (min_scan > 0 || (inactq_shortage > 0 && maxscan > 0)) vmd->vmd_last_active_scan = scan_tick; + SDT_PROBE3(vm, , , before__active__scan, vmd, pass, page_shortage); + /* * Scan the active queue for pages that can be deactivated. Update * the per-page activity counter and use it to identify deactivation @@ -1271,6 +1288,9 @@ relock_queue: vm_page_unlock(m); } vm_pagequeue_unlock(pq); + + SDT_PROBE3(vm, , , after__active__scan, vmd, pass, page_shortage); + #if !defined(NO_SWAPPING) /* * Idle process swapout -- run once per second. -- 2.11.0 From 320d166c11cef5b08a57c66ce01f9aa72f10d998 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Jun 2016 14:39:28 +0200 Subject: [PATCH 221/255] sys/amd64/conf/ELECTRO_BLOAT: Remove COMPAT support for FreeBSD versions before 10 Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BLOAT | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index a61f4e5eee3a..3bd60d2df610 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -48,11 +48,6 @@ options GEOM_PART_GPT # GUID Partition Tables. options GEOM_RAID # Soft RAID functionality. options GEOM_LABEL # Provides labelization options COMPAT_FREEBSD32 # Compatible with i386 binaries -options COMPAT_FREEBSD4 # Compatible with FreeBSD4 -options COMPAT_FREEBSD5 # Compatible with FreeBSD5 -options COMPAT_FREEBSD6 # Compatible with FreeBSD6 -options COMPAT_FREEBSD7 # Compatible with FreeBSD7 -options COMPAT_FREEBSD9 # Compatible with FreeBSD9 options COMPAT_FREEBSD10 # Compatible with FreeBSD10 options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI options KTRACE # ktrace(1) support -- 2.11.0 From 7b19aca7bb42cb79b927bb6316c1be2c780336b9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 13 Feb 2015 17:56:07 +0100 Subject: [PATCH 222/255] range_tree_destroy(): Optionally tolerate non-zero rt->rt_space Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c index 6422fd1c1fa6..842480a9e4ad 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c @@ -150,7 +150,11 @@ range_tree_create(range_tree_ops_t *ops, void *arg, kmutex_t *lp) void range_tree_destroy(range_tree_t *rt) { - VERIFY0(rt->rt_space); + if (rt->rt_space != 0) { + zfs_panic_recover("zfs: range_tree_destroy(): " + "rt->rt_space != 0: %llx", rt->rt_space); + return; + } if (rt->rt_ops != NULL) rt->rt_ops->rtop_destroy(rt, rt->rt_arg); -- 2.11.0 From f221972f775de8bcd492d9e8d76700afbc9089a6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 6 May 2015 14:04:57 +0200 Subject: [PATCH 223/255] zvol.c: Fix comment Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c index 23284a75eb16..36d88df2b79f 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c @@ -2282,7 +2282,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp) /* * If the caller really wants synchronous writes, and - * can't wait for them, don't return until the write + * can wait for them, don't return until the write * is done. */ if (df.df_flags & DF_WAIT_SYNC) { -- 2.11.0 From a9d01df8a94278085d805229f981507e913a5957 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 17 Jul 2015 13:11:39 +0200 Subject: [PATCH 224/255] Let the ZFS ARC behave better under memory pressure Original author: Karl Denninger Source: https://bz-attachments.freebsd.org/attachment.cgi?id=152852&action=diff&collapsed=&context=patch&format=raw&headers=1 PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594 Commit adjusted to compile on i386 and to deal with merge conflicts with (at least): r263620/6fcf6199a: "Rename global cnt to vm_cnt to avoid shadowing" r286625/78648874e: "5376 arc_kmem_reap_now() should not result in clearing arc_no_grow" r286763/71fb6300f: "5497 lock contention on arcs_mtx" No functional change intended. --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 82 +++++++++++++++++----- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 47 +++++++++++-- .../opensolaris/uts/common/fs/zfs/dsl_pool.c | 7 +- .../opensolaris/uts/common/fs/zfs/sys/dsl_pool.h | 1 + .../contrib/opensolaris/uts/common/fs/zfs/zio.c | 4 +- 5 files changed, 116 insertions(+), 25 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 6e9a624513cf..613a4ebdf1d1 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -341,6 +341,15 @@ static int arc_dead; extern boolean_t zfs_prefetch_disable; /* + * KD 2015-02-10 + * We have to be able to test for UIO use inside the arc allocator. + * NOTE: DO NOT MODIFY HERE! + */ +extern int zio_use_uma; +extern int zfs_dynamic_write_buffer; + + +/* * The arc has filled available memory and has now warmed up. */ static boolean_t arc_warm; @@ -373,7 +382,7 @@ static void arc_free_target_init(void *unused __unused) { - zfs_arc_free_target = vm_pageout_wakeup_thresh; + zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -394,6 +403,9 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW, "log2(fraction of arc to reclaim)"); SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN, &zfs_compressed_arc_enabled, 0, "Enable compressed ARC"); +SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, + &zfs_dynamic_write_buffer, 0, + "Dynamically restrict dirty data when memory is low"); /* * We don't have a tunable for arc_free_target due to the dependency on @@ -4011,13 +4023,32 @@ extern kmem_cache_t *zio_buf_cache[]; extern kmem_cache_t *zio_data_buf_cache[]; extern kmem_cache_t *range_seg_cache; -static __noinline void -arc_kmem_reap_now(void) +static void __used +reap_arc_caches() { size_t i; kmem_cache_t *prev_cache = NULL; kmem_cache_t *prev_data_cache = NULL; + for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { + if (zio_buf_cache[i] != prev_cache) { + prev_cache = zio_buf_cache[i]; + kmem_cache_reap_now(zio_buf_cache[i]); + } + if (zio_data_buf_cache[i] != prev_data_cache) { + prev_data_cache = zio_data_buf_cache[i]; + kmem_cache_reap_now(zio_data_buf_cache[i]); + } + } + kmem_cache_reap_now(buf_cache); + kmem_cache_reap_now(hdr_full_cache); + kmem_cache_reap_now(hdr_l2only_cache); + kmem_cache_reap_now(range_seg_cache); +} + +static __noinline void +arc_kmem_reap_now(void) +{ DTRACE_PROBE(arc__kmem_reap_start); #ifdef _KERNEL if (arc_meta_used >= arc_meta_limit) { @@ -4035,20 +4066,7 @@ arc_kmem_reap_now(void) #endif #endif - for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { - if (zio_buf_cache[i] != prev_cache) { - prev_cache = zio_buf_cache[i]; - kmem_cache_reap_now(zio_buf_cache[i]); - } - if (zio_data_buf_cache[i] != prev_data_cache) { - prev_data_cache = zio_data_buf_cache[i]; - kmem_cache_reap_now(zio_data_buf_cache[i]); - } - } - kmem_cache_reap_now(buf_cache); - kmem_cache_reap_now(hdr_full_cache); - kmem_cache_reap_now(hdr_l2only_cache); - kmem_cache_reap_now(range_seg_cache); + reap_arc_caches(); #ifdef illumos if (zio_arena != NULL) { @@ -4083,11 +4101,28 @@ arc_reclaim_thread(void *dummy __unused) { hrtime_t growtime = 0; callb_cpr_t cpr; + int autoreap = 0; CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG); mutex_enter(&arc_reclaim_lock); while (!arc_reclaim_thread_exit) { +#ifdef _KERNEL +/* KD 2015-02-10 + * Protect against UMA free memory bloat. We already do this on a low-memory + * basis in the allocator; it has to happen there rather than here due to + * response time considerations. Make the call here once every 10 passes as + * well; this reclaims unused UMA buffers every 10 seconds on an idle system + * and more frequently if the reclaim thread gets woken up by low RAM + * conditions. + */ + if ((zio_use_uma) && (autoreap++ == 10)) { + autoreap = 0; + DTRACE_PROBE(arc__reclaim_timed_reap); + reap_arc_caches(); + } +#endif /* _KERNEL */ + int64_t free_memory = arc_available_memory(); uint64_t evicted = 0; @@ -4373,6 +4408,19 @@ arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag) arc_space_consume(size, ARC_SPACE_META); } else { ASSERT(type == ARC_BUFC_DATA); +#ifdef _KERNEL +/* KD 2015-02-10 + * It would be nice if we could leave this to the arc_reclaim thread. + * Unfortunately we cannot; the test has to be done here as well, because + * under heavy I/O demand we can grab enough RAM fast enough to induce + * nasty oscillation problems. Fortunately we only need to call this when + * the system is under reasonably-severe memory stress. + */ + if (zio_use_uma && (ptob(vm_cnt.v_free_count) + size < ptob(vm_cnt.v_free_target))) { + DTRACE_PROBE3(arc__alloc_lowmem_reap, int, vm_cnt.v_free_count, int, size, int, vm_cnt.v_free_target); + reap_arc_caches(); + } +#endif /* _KERNEL */ datap = zio_data_buf_alloc(size); arc_space_consume(size, ARC_SPACE_DATA); } diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 68381488b38c..5bcc2cec49e3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -43,6 +43,8 @@ typedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn, uint64_t arg1, uint64_t arg2); +extern int zio_use_uma; /* Needs to be visible; DO NOT MODIFY! */ +int zfs_dynamic_write_buffer = 1; /* Dynamically tune writes */ dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) @@ -1106,7 +1108,7 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) { dsl_pool_t *dp = tx->tx_pool; uint64_t delay_min_bytes = - zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100; + zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; hrtime_t wakeup, min_tx_time, now; if (dirty <= delay_min_bytes) @@ -1118,11 +1120,11 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) * have to handle the case of it being >= the max, which could * cause a divide-by-zero if it's == the max. */ - ASSERT3U(dirty, <, zfs_dirty_data_max); + ASSERT3U(dirty, <, zfs_dirty_data_max_internal); now = gethrtime(); min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max - dirty); + (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); if (now > tx->tx_start + min_tx_time) return; @@ -1327,6 +1329,7 @@ int dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) { int err; + static uint64_t last_max; ASSERT(tx->tx_txg == 0); ASSERT(txg_how == TXG_WAIT || txg_how == TXG_NOWAIT || @@ -1339,6 +1342,42 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) if (txg_how == TXG_WAITED) tx->tx_waited = B_TRUE; +#ifdef _KERNEL + /* + * KD 2014-09-22 + * If UMA is enabled it can only return a previously-used block + * of identical size to what it had out before. If it's not the + * same size it will allocate a new one. This is a problem because + * dirty_data_max is the total dirty write data allowed out at any + * given time, but with UMA on that can multiply by the number of + * different block sizes (!!) requested in terms of free RAM that + * is left allocated but unused. For this reason never allow + * dirty_data_max to exceed the difference between the paging + * threshold and the current free memory, with a minimum of 256MB. + * This throttles "burst" allocations and prevents the system from + * choking during times of high write I/O demand. + * + * We allow this to be turned off if you want with + * "vfs.zfs_dynamic_write_buffer=0", which can be done in real time. + * + * Note that we work on the zfs_dirty_data_max_internal variable, + * because the user may set zfs_dirty_data_max himself and we must + * must honor that as a hard cap so it remains a usable tunable value. + */ + if (zio_use_uma & zfs_dynamic_write_buffer) { + zfs_dirty_data_max_internal = 1 << 28; + zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max_max); + if (last_max != (zfs_dirty_data_max_internal / (1024 * 1024))) { + last_max = zfs_dirty_data_max_internal / (1024 * 1024); + DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); + } + } else { + zfs_dirty_data_max_internal = zfs_dirty_data_max; + } +#endif /* _KERNEL */ + while ((err = dmu_tx_try_assign(tx, txg_how)) != 0) { dmu_tx_unassign(tx); @@ -1369,7 +1408,7 @@ dmu_tx_wait(dmu_tx_t *tx) * space. */ mutex_enter(&dp->dp_lock); - while (dp->dp_dirty_total >= zfs_dirty_data_max) + while (dp->dp_dirty_total >= zfs_dirty_data_max_internal) cv_wait(&dp->dp_spaceavail_cv, &dp->dp_lock); uint64_t dirty = dp->dp_dirty_total; mutex_exit(&dp->dp_lock); diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index 9b3b79bfb517..33820f76e7b3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -99,8 +99,11 @@ /* * zfs_dirty_data_max will be set to zfs_dirty_data_max_percent% of all memory, * capped at zfs_dirty_data_max_max. It can also be overridden in /etc/system. + * We also dynamically tune during low memory, honoring the sysctl set, so + * internal comparisons are against zfs_dirty_data_max_internal. */ uint64_t zfs_dirty_data_max; +uint64_t zfs_dirty_data_max_internal; uint64_t zfs_dirty_data_max_max = 4ULL * 1024 * 1024 * 1024; int zfs_dirty_data_max_percent = 10; @@ -548,7 +551,7 @@ dsl_pool_dirty_delta(dsl_pool_t *dp, int64_t delta) * Note: we signal even when increasing dp_dirty_total. * This ensures forward progress -- each thread wakes the next waiter. */ - if (dp->dp_dirty_total <= zfs_dirty_data_max) + if (dp->dp_dirty_total <= zfs_dirty_data_max_internal) cv_signal(&dp->dp_spaceavail_cv); } @@ -730,7 +733,7 @@ boolean_t dsl_pool_need_dirty_delay(dsl_pool_t *dp) { uint64_t delay_min_bytes = - zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100; + zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; boolean_t rv; mutex_enter(&dp->dp_lock); diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h index 0e27a538208c..ee97b5719cd2 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h @@ -50,6 +50,7 @@ struct dmu_tx; struct dsl_scan; extern uint64_t zfs_dirty_data_max; +extern uint64_t zfs_dirty_data_max_internal; extern uint64_t zfs_dirty_data_max_max; extern uint64_t zfs_dirty_data_sync; extern int zfs_dirty_data_max_percent; diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c index f8907c1e6872..d5ab54271619 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c @@ -46,9 +46,9 @@ SYSCTL_DECL(_vfs_zfs); SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); #if defined(__amd64__) -static int zio_use_uma = 1; +int zio_use_uma = 1; #else -static int zio_use_uma = 0; +int zio_use_uma = 0; #endif SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, use_uma, CTLFLAG_RDTUN, &zio_use_uma, 0, "Use uma(9) for ZIO allocations"); -- 2.11.0 From 691ab29afaba1f109d6b7ba3a8aea464d24205da Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 11:33:46 +0200 Subject: [PATCH 225/255] ZFS ARC: Add a missing & to get a logical and instead of a binary one Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 5bcc2cec49e3..6e89c7d3b6a5 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1364,7 +1364,7 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) * because the user may set zfs_dirty_data_max himself and we must * must honor that as a hard cap so it remains a usable tunable value. */ - if (zio_use_uma & zfs_dynamic_write_buffer) { + if (zio_use_uma && zfs_dynamic_write_buffer) { zfs_dirty_data_max_internal = 1 << 28; zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); -- 2.11.0 From 2a4c455cdb3087c8779a110bb81c3985b3d4523a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 3 Sep 2015 13:58:29 +0200 Subject: [PATCH 226/255] ZFS ARC: Move the ARC cache reaping out of the hot path ... and use a time-based rate limiting approach that can be controlled with the vfs.zfs.arc_reap_delay_min sysctl. Previously memory pressure could result in several thousand ARC reaping attempts per second. This wasn't merely inefficient but also caused lock contention and poor latency for ZFS operations that relied on arc_get_data_buf(). For details see: https://www.fabiankeil.de/gehacktes/electrobsd/zfs-arc-tuning/ Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 88 ++++++++++++++++------ 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 613a4ebdf1d1..8886ddcfeb89 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -4023,8 +4023,21 @@ extern kmem_cache_t *zio_buf_cache[]; extern kmem_cache_t *zio_data_buf_cache[]; extern kmem_cache_t *range_seg_cache; -static void __used -reap_arc_caches() +/* + * Used by arc_kmem_reap_now() and consider_reaping_arc_caches() + * to limit the time spent reaping. + * + * The arc_reaping_in_progress is a (somewhat racy) left over from a + * previous version of this code which could trigger multiple ARC cache + * reapings in parallel which should be avoided to reduce lock + * contention. It's hasn't been removed yet to encourage further + * experimenting. + */ +static unsigned int arc_reaping_in_progress = 0; +static sbintime_t last_reaping = 0; + +static void __noinline +reap_arc_caches(void) { size_t i; kmem_cache_t *prev_cache = NULL; @@ -4050,6 +4063,8 @@ static __noinline void arc_kmem_reap_now(void) { DTRACE_PROBE(arc__kmem_reap_start); + arc_reaping_in_progress++; + #ifdef _KERNEL if (arc_meta_used >= arc_meta_limit) { /* @@ -4077,9 +4092,49 @@ arc_kmem_reap_now(void) vmem_qcache_reap(zio_arena); } #endif +#ifdef _KERNEL + last_reaping = getsbinuptime(); +#endif + arc_reaping_in_progress = 0; DTRACE_PROBE(arc__kmem_reap_end); } + +/* + * Declared writable to allow resetting it. + * XXX: Should probably be a uint64 and integrated with kstat. + */ +static unsigned int arc_cache_reapings_skipped = 0; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_skipped, CTLFLAG_RW, + &arc_cache_reapings_skipped, 0, "Number of times the ARC caches have not been reaped due to the reap delay"); + +static unsigned int min_arc_reap_delay = 200; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_reap_delay_min, CTLFLAG_RW, + &min_arc_reap_delay, 200, "Minimum delay between ARC cache reapings (milliseconds)"); + +static void __noinline +consider_reaping_arc_caches(void) +{ +#ifdef _KERNEL + sbintime_t now; + + if (arc_reaping_in_progress) { + /* Already reaping in another thread. */ + arc_cache_reapings_skipped++; + return; + } + + now = getsbinuptime(); + if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) + { + /* Too soon to reap again. */ + arc_cache_reapings_skipped++; + return; + } +#endif + arc_kmem_reap_now(); +} + /* * Threads can block in arc_get_data_buf() waiting for this thread to evict * enough data and signal them to proceed. When this happens, the threads in @@ -4101,28 +4156,11 @@ arc_reclaim_thread(void *dummy __unused) { hrtime_t growtime = 0; callb_cpr_t cpr; - int autoreap = 0; CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG); mutex_enter(&arc_reclaim_lock); while (!arc_reclaim_thread_exit) { -#ifdef _KERNEL -/* KD 2015-02-10 - * Protect against UMA free memory bloat. We already do this on a low-memory - * basis in the allocator; it has to happen there rather than here due to - * response time considerations. Make the call here once every 10 passes as - * well; this reclaims unused UMA buffers every 10 seconds on an idle system - * and more frequently if the reclaim thread gets woken up by low RAM - * conditions. - */ - if ((zio_use_uma) && (autoreap++ == 10)) { - autoreap = 0; - DTRACE_PROBE(arc__reclaim_timed_reap); - reap_arc_caches(); - } -#endif /* _KERNEL */ - int64_t free_memory = arc_available_memory(); uint64_t evicted = 0; @@ -4153,8 +4191,6 @@ arc_reclaim_thread(void *dummy __unused) */ growtime = gethrtime() + SEC2NSEC(arc_grow_retry); - arc_kmem_reap_now(); - /* * If we are still low on memory, shrink the ARC * so that we have arc_shrink_min free space. @@ -4177,6 +4213,16 @@ arc_reclaim_thread(void *dummy __unused) evicted = arc_adjust(); + /* + * XXX: 2016-10-10: This was moved from arc_user_evicts_thread() + * which got deleted by the commit to keep the + * ARC data compressed. + * + * Consider reaping the ARC caches at least once per + * second, but more often when signalled under pressure. + */ + consider_reaping_arc_caches(); + mutex_enter(&arc_reclaim_lock); /* -- 2.11.0 From f187da27645f6a4b22caeca90fb4878406fae060 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Oct 2015 13:12:26 +0200 Subject: [PATCH 227/255] ZFS ARC: Ignore the reap delay when under memory pressure Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 8886ddcfeb89..55a1ca4c1ff3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -4107,6 +4107,9 @@ arc_kmem_reap_now(void) static unsigned int arc_cache_reapings_skipped = 0; SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_skipped, CTLFLAG_RW, &arc_cache_reapings_skipped, 0, "Number of times the ARC caches have not been reaped due to the reap delay"); +static unsigned int arc_cache_reapings_forced = 0; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_forced, CTLFLAG_RW, + &arc_cache_reapings_forced, 0, "Number of times the ARC caches reap delay was ignored due to memory pressure"); static unsigned int min_arc_reap_delay = 200; SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_reap_delay_min, CTLFLAG_RW, @@ -4125,11 +4128,17 @@ consider_reaping_arc_caches(void) } now = getsbinuptime(); - if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) - { - /* Too soon to reap again. */ - arc_cache_reapings_skipped++; - return; + if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) { + /* + * Skip the reaping unless there's memory pressure + * in which case we would risk vm deadlocks. We don't + * use vm_page_count_severe() as it triggers too late. + */ + if (!vm_page_count_min()) { + arc_cache_reapings_skipped++; + return; + } + arc_cache_reapings_forced++; } #endif arc_kmem_reap_now(); -- 2.11.0 From c2e355479e2879464602e151a6f2ab5ed84fe636 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 19:53:09 +0200 Subject: [PATCH 228/255] top: Show ZFS ARC target size Obtained from: ElectroBSD --- usr.bin/top/machine.c | 6 ++++-- usr.bin/top/top.local.1 | 12 +++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/usr.bin/top/machine.c b/usr.bin/top/machine.c index 795548075e1a..0bf3a85f202c 100644 --- a/usr.bin/top/machine.c +++ b/usr.bin/top/machine.c @@ -182,9 +182,9 @@ char *memorynames[] = { "K Free", NULL }; -int arc_stats[7]; +int arc_stats[8]; char *arcnames[] = { - "K Total, ", "K MFU, ", "K MRU, ", "K Anon, ", "K Header, ", "K Other", + "K Total, ", "K MFU, ", "K MRU, ", "K Anon, ", "K Header, ", "K Other, ", "K Target", NULL }; @@ -557,6 +557,8 @@ get_system_info(struct system_info *si) arc_stats[4] = arc_stat + arc_stat2 >> 10; GETSYSCTL("kstat.zfs.misc.arcstats.other_size", arc_stat); arc_stats[5] = arc_stat >> 10; + GETSYSCTL("kstat.zfs.misc.arcstats.c", arc_stat); + arc_stats[6] = arc_stat >> 10; si->arc = arc_stats; } diff --git a/usr.bin/top/top.local.1 b/usr.bin/top/top.local.1 index 864ab8d65c4b..951754c83bbe 100644 --- a/usr.bin/top/top.local.1 +++ b/usr.bin/top/top.local.1 @@ -2,9 +2,9 @@ .SH "FreeBSD NOTES" .SH DESCRIPTION OF MEMORY -Mem: 9220K Active, 1M Inact, 3284K Wired, 1M Cache, 2M Buf, 1320K Free -ARC: 2048K Total, 342K MRU, 760K MFU, 272K Anon, 232K Header, 442K Other -Swap: 91M Total, 79M Free, 13% Inuse, 80K In, 104K Out +Mem: 387M Active, 499M Inact, 945M Wired, 8K Cache, 5528K Buf, 83M Free +ARC: 349M Total, 174M MFU, 94M MRU, 2192K Anon, 6488K Header, 73M Other, 350M Target +Swap: 2048M Total, 124M Used, 1924M Free, 6% Inuse .TP .B K: Kilobyte @@ -57,6 +57,12 @@ number of ARC bytes holding headers .TP .B Other miscellaneous ARC bytes +.TP +.B Target +ARC target size, that is the total amount of memory +the ARC considers usable for itself. If it's not equal +to the total size, the ARC will shrink or grow to reach +the target. .SS Swap Stats .TP .B Total: -- 2.11.0 From 773bd0b8a9d5d15bc6142eafde155de869c0f93b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 25 Oct 2015 19:49:47 +0100 Subject: [PATCH 229/255] ZFS dsl_scan_visitds(): Don't panic if a device disappears while scrubbing Prevents: Unread portion of the kernel message buffer: [4299] panic: solaris assert: dmu_objset_find_dp(dp, dp->dp_root_dir_obj, enqueue_clones_cb, &eca, (1<<1)) == 0 (0x6 == 0x0), file: /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c, line: 1130 [4299] cpuid = 1 [4299] KDB: stack backtrace: [4299] db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe00949ed310 [4299] vpanic() at vpanic+0x182/frame 0xfffffe00949ed390 [4299] panic() at panic+0x43/frame 0xfffffe00949ed3f0 [4299] zfs_kmem_alloc() at zfs_kmem_alloc/frame 0xfffffe00949ed440 [4299] dsl_scan_visitds() at dsl_scan_visitds+0x551/frame 0xfffffe00949ed570 [4299] dsl_scan_visit() at dsl_scan_visit+0x22e/frame 0xfffffe00949ed790 [4299] dsl_scan_sync() at dsl_scan_sync+0x9da/frame 0xfffffe00949ed920 [4299] spa_sync() at spa_sync+0x564/frame 0xfffffe00949eda90 [4299] txg_sync_thread() at txg_sync_thread+0x3f1/frame 0xfffffe00949edbb0 [4299] fork_exit() at fork_exit+0x9c/frame 0xfffffe00949edbf0 [4299] fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00949edbf0 [4299] --- trap 0, rip = 0, rsp = 0, rbp = 0 --- [4299] KDB: enter: panic Note that this is not the only place where dmu_objset_find_dp() failures lead to panics and ENXIO probably isn't the only return code we have to expect anyway. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c index 4ebbc1a42050..a00cab366a4f 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c @@ -1179,12 +1179,14 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) scn->scn_phys.scn_queue_obj, dsl_dataset_phys(ds)->ds_creation_txg, tx)); } else { + int error; struct enqueue_clones_arg eca; eca.tx = tx; eca.originobj = ds->ds_object; - VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, - enqueue_clones_cb, &eca, DS_FIND_CHILDREN)); + error = dmu_objset_find_dp(dp, dp->dp_root_dir_obj, + enqueue_clones_cb, &eca, DS_FIND_CHILDREN); + VERIFY(error == 0 || error == ENXIO); } } -- 2.11.0 From d15d10ac74ed529406cf323638be4e95676e3929 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 27 Oct 2015 23:09:03 +0100 Subject: [PATCH 230/255] ZFS: Optionally let spa_sync() wait until at least one vdev is writable ... before calling dsl_pool_sync(). Significantly reduces the chances that a temporary loss of all vdevs results in a spa_namespace_lock deadlock that causes the pool to become inaccesible until the next reboot if the failmode is "wait" (default). The deadlock is caused by spa_sync() holding the required locks to fault or clear the pool while waiting for the sync operation to complete, thus preventing other parts of the system to get the pool in a usable state again. As a side effect of the deadlock, various zfs and zpool commands can lock up as well. Obtained from: ElectroBSD Deadlock example: 6 102135 zfskern txg_thread_enter mi_switch+0xde sleepq_wait+0x3a _cv_wait+0x1a6 zio_wait+0x89 dsl_pool_sync+0x12e spa_sync+0x51f txg_sync_thread+0x408 fork_exit+0x9a fork_trampoline+0xe 5985 101966 zpool - mi_switch+0xde sleepq_wait+0x3a _sx_xlock_hard+0x540 _sx_xlock+0x5d spa_get_errlog_size+0x3a spa_get_stats+0x116 zfs_ioc_pool_stats+0x4c zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 Xfast_syscall+0xfb 6863 101673 zpool - mi_switch+0xde sleepq_wait+0x3a _cv_wait+0x1a6 zio_wait+0x89 dbuf_read+0x464 dmu_buf_hold+0x9a zap_get_leaf_byblk+0x76 zap_deref_leaf+0xc6 fzap_cursor_retrieve+0x169 zap_cursor_retrieve+0x216 process_error_log+0xb1 spa_get_errlog+0xae zfs_ioc_error_log+0x70 zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 6885 100784 zpool - mi_switch+0xde sleepq_wait+0x3a _sx_xlock_hard+0x540 _sx_xlock+0x5d spa_get_errlog_size+0x3a spa_get_stats+0x116 zfs_ioc_pool_stats+0x4c zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 Xfast_syscall+0xfb See also: https://lists.freebsd.org/pipermail/freebsd-current/2014-September/052018.html Log excerpt with the sysctl enabled (single-disk pool): Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: g_eli_read_done() failed (error=5) label/prot1.eli[READ(offset=4654291968, length=9216)] Oct 28 12:27:03 r500 kernel: [415] da0 at umass-sim0 bus 0 scbus2 target 0 lun 0 Oct 28 12:27:03 r500 kernel: [415] da0: s/n 22B39DCC detached Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: Device label/prot1.eli destroyed. Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: Detached label/prot1.eli on last close. Oct 28 12:27:03 r500 kernel: [415] (da0:umass-sim0:0:0:0): Periph destroyed Oct 28 12:27:03 r500 ZFS: vdev is removed, pool_guid=2852801396507945726 vdev_guid=18325185749120546863 Oct 28 12:27:04 r500 kernel: [416] spa_sync: No writable vdev for prot1. Oct 28 12:27:05 r500 kernel: [417] spa_sync: No writable vdev for prot1. Oct 28 12:27:06 r500 kernel: [418] spa_sync: No writable vdev for prot1. Oct 28 12:27:07 r500 kernel: [419] spa_sync: No writable vdev for prot1. Oct 28 12:27:08 r500 kernel: [420] spa_sync: No writable vdev for prot1. Oct 28 12:27:09 r500 kernel: [420] umass0: on usbus1 Oct 28 12:27:09 r500 kernel: [420] umass0: SCSI over Bulk-Only; quirks = 0x4101 Oct 28 12:27:09 r500 kernel: [420] umass0:2:0: Attached to scbus2 Oct 28 12:27:09 r500 kernel: [420] da0 at umass-sim0 bus 0 scbus2 target 0 lun 0 Oct 28 12:27:09 r500 kernel: [420] da0: Removable Direct Access SCSI-2 device Oct 28 12:27:09 r500 kernel: [420] da0: Serial Number 22B39DCC Oct 28 12:27:09 r500 kernel: [420] da0: 40.000MB/s transfers Oct 28 12:27:09 r500 kernel: [420] da0: 7782MB (15937536 512 byte sectors) Oct 28 12:27:09 r500 kernel: [420] da0: quirks=0x2 Oct 28 12:27:09 r500 kernel: [421] GEOM_PART: integrity check failed (label/prot1, MBR) Oct 28 12:27:09 r500 kernel: [421] spa_sync: No writable vdev for prot1. Oct 28 12:27:10 r500 kernel: [422] spa_sync: No writable vdev for prot1. Oct 28 12:27:11 r500 kernel: [423] spa_sync: No writable vdev for prot1. Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Device label/prot1.eli created. Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Encryption: AES-XTS 256 Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Crypto: software Oct 28 12:27:12 r500 ZFS: vdev state changed, pool_guid=2852801396507945726 vdev_guid=18325185749120546863 Oct 28 12:27:12 r500 kernel: [424] spa_sync: vdev for prot1 became writable again. Device loss was triggered and fixed with: sudo usbconfig -d 1.2 reset && zogftw import prot1 && sudo zpool clear prot1 This commit is experimental! It should be a nop unless the vfs.zfs.spa_sync_wait_for_writable_vdev sysctl is set, though. Setting the sysctl merely shrinks the race window and thus does not completely solve the problem. The problem can still be reproduced with an artifical test case like: Shell 1: mdconfig -u 0 -f /dpool/scratch/test-vdev.img zpool create test /dev/md0 while sleep 1; do mdconfig -d -u 0 -o force && mdconfig -f /dpool/scratch/test-vdev.img && zpool clear test; done Shell 2: # Cause writes to the pool from another shell, for example # by creating datasets. Log excerpt (from test begin to deadlock): Oct 29 12:34:28 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:44:42 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:45:04 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:43 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:43 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:44 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:44 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:45 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:45 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:46 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:46 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:47 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:47 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:48 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:48 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:49 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:49 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:51 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:51 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:52 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:52 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:53 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:54 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:54 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:55 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:55 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:56 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:56 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:57 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:57 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:59 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:59 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:47:00 kendra kernel: g_dev_taste: make_dev_p() failed (gp->name=md0, error=17) With the deadman enabled, this will also cause: panic: I/O to pool 'test' appears to be hung on vdev guid 3080051161477470469 at '/dev/md0'. cpuid = 0 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe01136af870 vpanic() at vpanic+0x182/frame 0xfffffe01136af8f0 panic() at panic+0x43/frame 0xfffffe01136af950 vdev_deadman() at vdev_deadman+0x127/frame 0xfffffe01136af9a0 vdev_deadman() at vdev_deadman+0x40/frame 0xfffffe01136af9f0 spa_deadman() at spa_deadman+0x86/frame 0xfffffe01136afa20 softclock_call_cc() at softclock_call_cc+0x1a3/frame 0xfffffe01136afaf0 softclock() at softclock+0x94/frame 0xfffffe01136afb20 intr_event_execute_handlers() at intr_event_execute_handlers+0x1b6/frame 0xfffffe01136afb60 ithread_loop() at ithread_loop+0xa6/frame 0xfffffe01136afbb0 fork_exit() at fork_exit+0x9c/frame 0xfffffe01136afbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01136afbf0 With test's txg_sync_thread being the offender: (kgdb) tid 101874 [Switching to thread 819 (Thread 101874)]#0 sched_switch (td=0xfffff800513649a0, newtd=, flags=) at /usr/src/sys/kern/sched_ule.c:1969 1969 cpuid = PCPU_GET(cpuid); (kgdb) where #0 sched_switch (td=0xfffff800513649a0, newtd=, flags=) at /usr/src/sys/kern/sched_ule.c:1969 #1 0xffffffff805a3a18 in mi_switch (flags=260, newtd=0x0) at /usr/src/sys/kern/kern_synch.c:470 #2 0xffffffff805ea15a in sleepq_wait (wchan=0x0, pri=0) at /usr/src/sys/kern/subr_sleepqueue.c:631 #3 0xffffffff80530509 in _cv_wait (cvp=0xfffff8002678ea98, lock=0xfffff8002678ea78) at /usr/src/sys/kern/kern_condvar.c:139 #4 0xffffffff81930bbb in zio_wait (zio=) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:1535 #5 0xffffffff818e4871 in dsl_pool_sync (dp=0xfffff80047dfd000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c:540 #6 0xffffffff81903653 in spa_sync (spa=0xfffff8009dfe2000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6734 #7 0xffffffff8190ccfa in txg_sync_thread (arg=0xfffff80047dfd000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:517 #8 0xffffffff80556edc in fork_exit (callout=0xffffffff8190c970 , arg=0xfffff80047dfd000, frame=0xfffffe011c27bc00) at /usr/src/sys/kern/kern_fork.c:1011 #9 0xffffffff8085b91e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:609 #10 0x0000000000000000 in ?? () (kgdb) f 6 #6 0xffffffff81903653 in spa_sync (spa=0xfffff8009dfe2000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6734 (kgdb) p spa->spa_name $3 = 0xfffff8009dfe2000 "test" --- .../contrib/opensolaris/uts/common/fs/zfs/spa.c | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c index fff5f8fadbef..54cb224a7cdd 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c @@ -6817,6 +6817,31 @@ spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) rrw_exit(&dp->dp_config_rwlock, FTAG); } + +/* + * Check if there's at least one (potentionally) writable vdev. + */ +int +spa_has_writable_vdev(spa_t *spa) { + vdev_t *rvd = spa->spa_root_vdev; + vdev_t *vd; + int i; + + for (i = 0; i < rvd->vdev_children; i++) { + vd = rvd->vdev_child[i]; + if (!vd->vdev_cant_write) + return 1; + } + return 0; +} + +static int spa_sync_wait_for_writable_vdev = 0; +SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_sync_wait_for_writable_vdev, CTLFLAG_RWTUN, + &spa_sync_wait_for_writable_vdev, 0, + "Let spa_sync() pause if no writable vdev is available. Experimental!"); + +static int spa_sync_deadlock_loops = 0; + /* * Sync the specified transaction group. New blocks may be dirtied as * part of the process, so we iterate until it converges. @@ -6950,6 +6975,29 @@ spa_sync(spa_t *spa, uint64_t txg) spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); spa_errlog_sync(spa, txg); + if (spa_sync_wait_for_writable_vdev) { + int waited_for_vdev = 0; + + while (!spa_has_writable_vdev(spa)) { + spa_config_exit(spa, SCL_CONFIG, FTAG); + + waited_for_vdev = 1; + printf("%s: No writable vdev for %s.\n", + __func__, spa_name(spa)); +#ifdef _KERNEL + pause("spa_sync", hz); +#else + sleep(1); +#endif + spa_sync_deadlock_loops++; + + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + } + if (waited_for_vdev) { + printf("%s: vdev for %s became writable again.\n", + __func__, spa_name(spa)); + } + } dsl_pool_sync(dp, txg); if (pass < zfs_sync_pass_deferred_free) { -- 2.11.0 From 65faefd9975611a5ec94ac6a2c717a8e1f4cf574 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 8 Jan 2016 13:30:21 +0100 Subject: [PATCH 231/255] ZFS ARC: Sync with FreeBSD bug #187594 (ZFS ARC behavior problem and fix) Main patch author: Karl Denninger PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594 PATCH URL: https://bz-attachments.freebsd.org/attachment.cgi?id=164051&action=diff&format=raw&headers=1 --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 60 +++++++++++++++++++++- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++- .../contrib/opensolaris/uts/common/fs/zfs/zio.c | 3 ++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 55a1ca4c1ff3..1a55d5ceca01 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -366,6 +366,14 @@ int zfs_arc_shrink_shift = 0; int zfs_arc_p_min_shift = 0; uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ u_int zfs_arc_free_target = 0; +u_int zfs_arc_wakeup_pager = 0; +u_int zfs_arc_wakeup_delay = 500; + +#define WAKE_PAGER +#ifdef WAKE_PAGER +#define WAKE_PAGER_CONSTANT 10 / 9 /* Pager wakeup threshold */ +static int arc_init_done = 0; /* We know arc_warm is valid */ +#endif /* WAKE_PAGER */ /* Absolute min for arc min / max is 16MB. */ static uint64_t arc_abs_min = 16 << 20; @@ -383,6 +391,9 @@ arc_free_target_init(void *unused __unused) { zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); +#ifdef WAKE_PAGER + zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; +#endif /* WAKE_PAGER */ } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -406,6 +417,12 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN, SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, &zfs_dynamic_write_buffer, 0, "Dynamically restrict dirty data when memory is low"); +#ifdef WAKE_PAGER +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_pager, CTLFLAG_RWTUN, + &zfs_arc_wakeup_pager, 0, "Wake VM below this number of pages"); +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_delay, CTLFLAG_RWTUN, + &zfs_arc_wakeup_delay, 0, "May wake up VM once this number of MS"); +#endif /* WAKE_PAGER */ /* * We don't have a tunable for arc_free_target due to the dependency on @@ -433,6 +450,9 @@ sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS) return (EINVAL); zfs_arc_free_target = val; +#ifdef WAKE_PAGER + zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; +#endif /* WAKE_PAGER */ return (0); } @@ -3866,6 +3886,11 @@ int64_t arc_pages_pp_reserve = 64; int64_t arc_swapfs_reserve = 64; /* + * Declare file-local static for event processor bypass + */ +static unsigned int arc_no_wake_event = 0; + +/* * Return the amount of memory that can be consumed before reclaim will be * needed. Positive if there is sufficient free memory, negative indicates * the amount of memory that needs to be freed up. @@ -3878,6 +3903,10 @@ arc_available_memory(void) free_memory_reason_t r = FMR_UNKNOWN; #ifdef _KERNEL +#ifdef WAKE_PAGER + sbintime_t now; + static sbintime_t last_pagedaemon_wake = 0; +#endif /* WAKE_PAGER */ if (needfree > 0) { n = PAGESIZE * (-needfree); if (n < lowest) { @@ -3896,6 +3925,26 @@ arc_available_memory(void) r = FMR_LOTSFREE; } +#ifdef WAKE_PAGER +/* + * If memory is less than the ARC wakeup threshold and time has expired since + * the last time we woke the pager... Do not execute until the ARC warms up. + */ + if ((arc_init_done) && + (((int64_t) freemem - zfs_arc_wakeup_pager) < 0) && + (arc_warm == B_TRUE) + ) { + now = getsbinuptime(); + if ((now - last_pagedaemon_wake) / SBT_1MS > zfs_arc_wakeup_delay) { + last_pagedaemon_wake = now; + arc_no_wake_event++; /* Set bypass flag for ARC */ + DTRACE_PROBE(arc__wake_pagedaemon); + pagedaemon_wakeup(); /* Wake the pager */ + } + } + +#endif /* WAKE_PAGER */ + #ifdef illumos /* * check that we're out of range of the pageout scanner. It starts to @@ -4004,6 +4053,7 @@ arc_available_memory(void) last_free_memory = lowest; last_free_reason = r; DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r); + return (lowest); } @@ -4034,9 +4084,10 @@ extern kmem_cache_t *range_seg_cache; * experimenting. */ static unsigned int arc_reaping_in_progress = 0; +static unsigned int arc_pagedaemon_ignore = 0; static sbintime_t last_reaping = 0; -static void __noinline +static __noinline void reap_arc_caches(void) { size_t i; @@ -5985,6 +6036,10 @@ static void arc_lowmem(void *arg __unused, int howto __unused) { + if (arc_no_wake_event) { /* Don't do it if we woke the pager */ + arc_no_wake_event = 0; /* Just clear the flag */ + return; + } mutex_enter(&arc_reclaim_lock); /* XXX: Memory deficit should be passed as argument. */ needfree = btoc(arc_c >> arc_shrink_shift); @@ -6299,6 +6354,9 @@ arc_init(void) printf(" in /boot/loader.conf.\n"); } #endif +#ifdef WAKE_PAGER + arc_init_done++; /* For anyone who wants to know */ +#endif /* WAKE_PAGER */ } void diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 6e89c7d3b6a5..691a651c8ef7 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1123,8 +1123,13 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) ASSERT3U(dirty, <, zfs_dirty_data_max_internal); now = gethrtime(); - min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + if (dirty >= zfs_dirty_data_max_internal) {/* No scaling if overcommitted */ + min_tx_time = zfs_delay_scale * + (dirty - delay_min_bytes); + } else { + min_tx_time = zfs_delay_scale * + (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + } if (now > tx->tx_start + min_tx_time) return; diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c index d5ab54271619..41664263f91a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c @@ -45,6 +45,9 @@ SYSCTL_DECL(_vfs_zfs); SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); +/* KD 2015-07-15 Change class to "int" from "static int" as we reference + * this as an extern elsewhere + */ #if defined(__amd64__) int zio_use_uma = 1; #else -- 2.11.0 From 306ff4a46572089abefc968da88131c094606584 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 17:07:24 +0200 Subject: [PATCH 232/255] sys/cddl: Allow to modify the ZFS deadman sysctls after the system is up There does not seem to be a technical reason why it shouldn't be done and being able to temporarily disable the deadman is useful when importing a pool that is backed by remote storage (such as ggated running as onion service). Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c index c247388468a6..c3fc2c979037 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c @@ -357,13 +357,13 @@ SYSCTL_PROC(_vfs_zfs, OID_AUTO, debug_flags, CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), sysctl_vfs_zfs_debug_flags, "IU", "Debug flags for ZFS testing."); -SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RDTUN, +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RWTUN, &zfs_deadman_synctime_ms, 0, "Stalled ZFS I/O expiration time in milliseconds"); -SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RDTUN, +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RWTUN, &zfs_deadman_checktime_ms, 0, "Period of checks for stalled ZFS I/O in milliseconds"); -SYSCTL_INT(_vfs_zfs, OID_AUTO, deadman_enabled, CTLFLAG_RDTUN, +SYSCTL_INT(_vfs_zfs, OID_AUTO, deadman_enabled, CTLFLAG_RWTUN, &zfs_deadman_enabled, 0, "Kernel panic on stalled ZFS I/O"); SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_asize_inflation, CTLFLAG_RWTUN, &spa_asize_inflation, 0, "Worst case inflation factor for single sector writes"); -- 2.11.0 From fc0df043a76eb936aa91a40445988ff57472a15d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 Jan 2016 13:32:24 +0100 Subject: [PATCH 233/255] ZFS ARC: If there's less than 1 GB of RAM, set the ARC max to 50% of it Now the code matches the comment again. The limit was increased to 6/8 in r172130 and reduced to 5/8 in r175633 again to mitigate 'kmem_map too small panics'. As it turns out, 5/8 is still too much and can result in deadlocks caused by the vm system running out of pages: (kgdb) p vm_cnt $7 = {v_swtch = 0, v_trap = 0, v_syscall = 0, v_intr = 0, v_soft = 0, v_vm_faults = 0, v_io_faults = 0, v_cow_faults = 0, v_cow_optim = 0, v_zfod = 0, v_ozfod = 0, v_swapin = 0, v_swapout = 0, v_swappgsin = 0, v_swappgsout = 0, v_vnodein = 0, v_vnodeout = 0, v_vnodepgsin = 0, v_vnodepgsout = 0, v_intrans = 0, v_reactivated = 0, v_pdwakeups = 878, v_pdpages = 0, v_tcached = 0, v_dfree = 0, v_pfree = 0, v_tfree = 0, v_page_size = 4096, v_page_count = 247933, v_free_reserved = 372, v_free_target = 5324, v_free_min = 1610, v_free_count = 2, v_wire_count = 72901, v_active_count = 174634, v_inactive_target = 7986, v_inactive_count = 395, v_cache_count = 0, v_pageout_free_min = 34, v_interrupt_free_min = 2, v_free_severe = 991, v_forks = 0, v_vforks = 0, v_rforks = 0, v_kthreads = 0, v_forkpages = 0, v_vforkpages = 0, v_rforkpages = 0, v_kthreadpages = 0, v_spare = 0xffffffff8141770c} Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 1a55d5ceca01..5e80d2c93439 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -6203,7 +6203,7 @@ arc_init(void) arc_c_max = (arc_c * 8) - (1 << 30); else arc_c_max = arc_c_min; - arc_c_max = MAX(arc_c * 5, arc_c_max); + arc_c_max = MAX(arc_c * 4, arc_c_max); /* * In userland, there's only the memory pressure that we artificially -- 2.11.0 From 9c53650b5c7b0709eab314bfef33ae4ac1b0c035 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 Jan 2016 12:44:01 +0100 Subject: [PATCH 234/255] ZFS ARC: Remove the 'WAKE_PAGER' ... as I suspect that the "Don't do it if we woke the pager" code in arc_lowmem() increase the chances that the vm runs out of free pages. It's also not obvious (to me) that the WAKE_PAGER does anything useful. Quoting https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594#c244: "I have not yet tried Karl's latest patch but like to point out that some of the underlying assumptions about how the vm pager behaves seem incorrect to me. For example I'd expect calling pagedaemon_wakeup() without memory pressure (from the pagers point of view) to be pretty close to a nop as vm_pageout_worker() does its own checks before doing any heavy lifting: http://fxr.watson.org/fxr/source/vm/vm_pageout.c#L1634 Also note that vm_pageout_worker() is already called at least once per second anyway: [fk@polizei-erziehung ~]$ sudo /usr/src/share/dtrace/monitor-page-scanner 2015 Nov 1 17:20:45: Monitoring the page scanner. Minimum pass value to show 'boring' scans without memory pressure or inactive page surplus: 2 (Launder dirty pages). Press CTRL-C to abort. 2015 Nov 1 17:21:45: Scan goals in the previous minute: Update active LRU/deactivate pages 60 2015 Nov 1 17:22:45: Scan goals in the previous minute: Update active LRU/deactivate pages 60 I'm not claiming that increasing the frequency when there's no memory pressure causes any harm (besides code complexity), but I'm not convinced that it has the intended effect and needs to be triggered from ZFS (as opposed to changing the pager defaults)." Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 45 ---------------------- 1 file changed, 45 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 5e80d2c93439..9c844e30e9e5 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -369,12 +369,6 @@ u_int zfs_arc_free_target = 0; u_int zfs_arc_wakeup_pager = 0; u_int zfs_arc_wakeup_delay = 500; -#define WAKE_PAGER -#ifdef WAKE_PAGER -#define WAKE_PAGER_CONSTANT 10 / 9 /* Pager wakeup threshold */ -static int arc_init_done = 0; /* We know arc_warm is valid */ -#endif /* WAKE_PAGER */ - /* Absolute min for arc min / max is 16MB. */ static uint64_t arc_abs_min = 16 << 20; @@ -391,9 +385,6 @@ arc_free_target_init(void *unused __unused) { zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); -#ifdef WAKE_PAGER - zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; -#endif /* WAKE_PAGER */ } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -417,12 +408,6 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN, SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, &zfs_dynamic_write_buffer, 0, "Dynamically restrict dirty data when memory is low"); -#ifdef WAKE_PAGER -SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_pager, CTLFLAG_RWTUN, - &zfs_arc_wakeup_pager, 0, "Wake VM below this number of pages"); -SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_delay, CTLFLAG_RWTUN, - &zfs_arc_wakeup_delay, 0, "May wake up VM once this number of MS"); -#endif /* WAKE_PAGER */ /* * We don't have a tunable for arc_free_target due to the dependency on @@ -450,9 +435,6 @@ sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS) return (EINVAL); zfs_arc_free_target = val; -#ifdef WAKE_PAGER - zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; -#endif /* WAKE_PAGER */ return (0); } @@ -3903,10 +3885,6 @@ arc_available_memory(void) free_memory_reason_t r = FMR_UNKNOWN; #ifdef _KERNEL -#ifdef WAKE_PAGER - sbintime_t now; - static sbintime_t last_pagedaemon_wake = 0; -#endif /* WAKE_PAGER */ if (needfree > 0) { n = PAGESIZE * (-needfree); if (n < lowest) { @@ -3925,26 +3903,6 @@ arc_available_memory(void) r = FMR_LOTSFREE; } -#ifdef WAKE_PAGER -/* - * If memory is less than the ARC wakeup threshold and time has expired since - * the last time we woke the pager... Do not execute until the ARC warms up. - */ - if ((arc_init_done) && - (((int64_t) freemem - zfs_arc_wakeup_pager) < 0) && - (arc_warm == B_TRUE) - ) { - now = getsbinuptime(); - if ((now - last_pagedaemon_wake) / SBT_1MS > zfs_arc_wakeup_delay) { - last_pagedaemon_wake = now; - arc_no_wake_event++; /* Set bypass flag for ARC */ - DTRACE_PROBE(arc__wake_pagedaemon); - pagedaemon_wakeup(); /* Wake the pager */ - } - } - -#endif /* WAKE_PAGER */ - #ifdef illumos /* * check that we're out of range of the pageout scanner. It starts to @@ -6354,9 +6312,6 @@ arc_init(void) printf(" in /boot/loader.conf.\n"); } #endif -#ifdef WAKE_PAGER - arc_init_done++; /* For anyone who wants to know */ -#endif /* WAKE_PAGER */ } void -- 2.11.0 From 9e6049941062ae3a172051c171b57ea3982b4381 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 2 May 2016 12:05:41 +0200 Subject: [PATCH 235/255] ZFS: Fix a typo in the delay_min_dirty_percent sysctl description Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index 33820f76e7b3..bcf9b055f96c 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -165,7 +165,7 @@ static int sysctl_zfs_delay_min_dirty_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs, OID_AUTO, delay_min_dirty_percent, CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(int), sysctl_zfs_delay_min_dirty_percent, "I", - "The limit of outstanding dirty data before transations are delayed"); + "The limit of outstanding dirty data before transactions are delayed"); static int sysctl_zfs_delay_scale(SYSCTL_HANDLER_ARGS); /* No zfs_delay_scale tunable due to limit requirements */ -- 2.11.0 From 4e27c09c1cfafd5ab38053bc03f02ab96d2c965a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 Apr 2016 09:41:25 +0200 Subject: [PATCH 236/255] ZFS: Make dmu_tx_delay() immune to zfs_dirty_data_max_internal changes while it's running as it can result in underflows and possible deadlocks Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 691a651c8ef7..bc006edc0f95 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1107,10 +1107,11 @@ static void dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) { dsl_pool_t *dp = tx->tx_pool; - uint64_t delay_min_bytes = - zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; + uint64_t delay_min_bytes; hrtime_t wakeup, min_tx_time, now; + uint64_t dirty_data_max = zfs_dirty_data_max_internal; + delay_min_bytes = dirty_data_max * zfs_delay_min_dirty_percent / 100; if (dirty <= delay_min_bytes) return; @@ -1120,15 +1121,15 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) * have to handle the case of it being >= the max, which could * cause a divide-by-zero if it's == the max. */ - ASSERT3U(dirty, <, zfs_dirty_data_max_internal); + ASSERT3U(dirty, <, dirty_data_max); now = gethrtime(); - if (dirty >= zfs_dirty_data_max_internal) {/* No scaling if overcommitted */ + if (dirty >= dirty_data_max) {/* No scaling if overcommitted */ min_tx_time = zfs_delay_scale * (dirty - delay_min_bytes); } else { min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + (dirty - delay_min_bytes) / (dirty_data_max - dirty); } if (now > tx->tx_start + min_tx_time) return; -- 2.11.0 From 549e56849adf97ae9e72042d6ef25e6a57fd5dfc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 5 May 2016 13:59:47 +0200 Subject: [PATCH 237/255] share/skel: Detach skelleton files for csh and rsh from the build Obtained from: ElectroBSD --- share/skel/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/skel/Makefile b/share/skel/Makefile index 6b67d2bdbec5..8c51c43c3439 100644 --- a/share/skel/Makefile +++ b/share/skel/Makefile @@ -2,9 +2,9 @@ # $FreeBSD$ FILESGROUPS= FILES1 FILES2 -FILES1= dot.cshrc dot.login dot.login_conf dot.mailrc dot.profile \ +FILES1= dot.login_conf dot.mailrc dot.profile \ dot.shrc -FILES2= dot.mail_aliases dot.rhosts +FILES2= dot.mail_aliases FILES1DIR= ${SHAREDIR}/skel FILES2DIR= ${SHAREDIR}/skel FILES1MODE= 0644 -- 2.11.0 From 4bbbcd98ea7a3c59778d17813124eee8243932db Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Apr 2016 10:48:23 +0200 Subject: [PATCH 238/255] ZFS: Make the dynamic write buffer more effective for memory-constrained systems - Remove the previously hardcoded 256 MB lower limit which is far too much for systems with 2 GB of RAM or less. - Prevent an integer underflow if the free count is below the free target. The vm seems to treat vm_cnt.v_free_target more like a guide line than a rule, and it's unsafe to assume that it will always be below the vm_cnt.v_free_count. (kgdb) p zfs_dirty_data_max_internal $8 = 10000000 (kgdb) p vm_cnt.v_free_count - vm_cnt.v_free_target $9 = 18446744073709546294 (kgdb) p vm_cnt.v_free_count $10 = 2 (kgdb) p vm_cnt.v_free_target $11 = 5324 If the free count is below the target, use half the free count as limit instead. There is probably a more appropriate limit, but for now this has to do. Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index bc006edc0f95..f1a0688cb1b3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1350,16 +1350,16 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) #ifdef _KERNEL /* - * KD 2014-09-22 * If UMA is enabled it can only return a previously-used block - * of identical size to what it had out before. If it's not the + * of identical size to what it had out before. If it's not the * same size it will allocate a new one. This is a problem because * dirty_data_max is the total dirty write data allowed out at any * given time, but with UMA on that can multiply by the number of * different block sizes (!!) requested in terms of free RAM that * is left allocated but unused. For this reason never allow * dirty_data_max to exceed the difference between the paging - * threshold and the current free memory, with a minimum of 256MB. + * threshold and the current free memory. + * * This throttles "burst" allocations and prevents the system from * choking during times of high write I/O demand. * @@ -1368,13 +1368,17 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) * * Note that we work on the zfs_dirty_data_max_internal variable, * because the user may set zfs_dirty_data_max himself and we must - * must honor that as a hard cap so it remains a usable tunable value. + * honor that as a hard cap so it remains a usable tunable value. */ if (zio_use_uma && zfs_dynamic_write_buffer) { - zfs_dirty_data_max_internal = 1 << 28; - zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max_max); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); + if (vm_cnt.v_free_count > vm_cnt.v_free_target) { + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, + ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); + } else { + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, + ptob(vm_cnt.v_free_count / 2)); + } if (last_max != (zfs_dirty_data_max_internal / (1024 * 1024))) { last_max = zfs_dirty_data_max_internal / (1024 * 1024); DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); -- 2.11.0 From dd1058dabf607656f6c81960251ffe1b9bd00453 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Apr 2016 11:21:17 +0200 Subject: [PATCH 239/255] ZFS: Add sysctl to show how often the write buffer was limitted Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index f1a0688cb1b3..6cc268812cde 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -46,6 +46,12 @@ typedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn, extern int zio_use_uma; /* Needs to be visible; DO NOT MODIFY! */ int zfs_dynamic_write_buffer = 1; /* Dynamically tune writes */ +SYSCTL_DECL(_vfs_zfs); +static uint64_t zfs_dynamic_write_buffer_hits = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dynamic_write_buffer_hits, CTLFLAG_RWTUN, + &zfs_dynamic_write_buffer_hits, 0, + "Number of times the dynamic write buffer was limitted"); + dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) { @@ -1383,6 +1389,9 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) last_max = zfs_dirty_data_max_internal / (1024 * 1024); DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); } + if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { + zfs_dynamic_write_buffer_hits++; + } } else { zfs_dirty_data_max_internal = zfs_dirty_data_max; } -- 2.11.0 From 023a4a1ed33f3216bfa768723666fea710c7f7d9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Apr 2016 11:52:35 +0200 Subject: [PATCH 240/255] ZFS: Add sysctl to read the dirty_data_max_internal Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index bcf9b055f96c..d99323fd21a3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -150,6 +150,10 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_max, CTLFLAG_RDTUN, &zfs_dirty_data_max_max, 0, "The absolute cap on dirty_data_max when auto calculating"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_internal, CTLFLAG_RD, + &zfs_dirty_data_max_internal, 0, + "The currently active limit of dirty data (may dynamically change)"); + static int sysctl_zfs_dirty_data_max_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs, OID_AUTO, dirty_data_max_percent, CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), -- 2.11.0 From 2bdce1a5edf9b8b58e800f5dd9673d56432fb6ec Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 28 Apr 2016 13:52:18 +0200 Subject: [PATCH 241/255] ZFS: Add a sysctl to access the lowest write buffer limit Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 6cc268812cde..51ded24b981f 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -52,6 +52,11 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dynamic_write_buffer_hits, CTLFLAG_RWTUN, &zfs_dynamic_write_buffer_hits, 0, "Number of times the dynamic write buffer was limitted"); +static uint64_t zfs_lowest_dynamic_write_buffer_limit = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, lowest_dynamic_write_buffer_limit, CTLFLAG_RD, + &zfs_lowest_dynamic_write_buffer_limit, 0, + "The lowest value the dynamic write buffer was limitted to"); + dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) { @@ -1392,6 +1397,10 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { zfs_dynamic_write_buffer_hits++; } + if (zfs_lowest_dynamic_write_buffer_limit == 0 || + zfs_lowest_dynamic_write_buffer_limit >= zfs_dirty_data_max_internal) { + zfs_lowest_dynamic_write_buffer_limit = zfs_dirty_data_max_internal; + } } else { zfs_dirty_data_max_internal = zfs_dirty_data_max; } -- 2.11.0 From 0e62c3639ede2c52274a17bc2ff5ad316a62e5a9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 11 May 2016 12:41:12 +0200 Subject: [PATCH 242/255] ZFS: Attempt to let the spa deadman store the last and the maximum vdev sync delay As it's useful to know when tuning the timeout. XXX: This looks like it should work, but actually doesn't. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c index 2d8d93fc8e76..3566c2b45f63 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c @@ -91,6 +91,16 @@ SYSCTL_NODE(_vfs_zfs, OID_AUTO, vdev, CTLFLAG_RW, 0, "ZFS VDEV"); static uint64_t zfs_max_auto_ashift = SPA_MAXASHIFT; static uint64_t zfs_min_auto_ashift = SPA_MINASHIFT; +static uint64_t zfs_max_vdev_sync_delay = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, vdev_sync_delay_max, CTLFLAG_RWTUN, + &zfs_max_vdev_sync_delay, 0, + "Maximum vdev sync delay seen by the spa deadman"); + +static uint64_t zfs_last_vdev_sync_delay = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, vdev_sync_delay_last, CTLFLAG_RWTUN, + &zfs_last_vdev_sync_delay, 0, + "Last vdev sync delay seen seen by the spa deadman"); + static int sysctl_vfs_zfs_max_auto_ashift(SYSCTL_HANDLER_ARGS) { @@ -3616,6 +3626,9 @@ vdev_deadman(vdev_t *vd) */ fio = avl_first(&vq->vq_active_tree); delta = gethrtime() - fio->io_timestamp; + zfs_last_vdev_sync_delay = delta; + if (zfs_max_vdev_sync_delay < delta) + zfs_max_vdev_sync_delay = delta; if (delta > spa_deadman_synctime(spa)) { zfs_dbgmsg("SLOW IO: zio timestamp %lluns, " "delta %lluns, last io %lluns", -- 2.11.0 From ff483c41d5930345649c2d3c8bf3b637b5f7d72a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 31 May 2016 23:46:19 +0200 Subject: [PATCH 243/255] ZFS: Use https for the error message URLs Obtained from: ElectroBSD --- cddl/contrib/opensolaris/cmd/zpool/zpool_main.c | 6 +++--- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c index e2dc24c5fa91..1ed99c9b0c22 100644 --- a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c +++ b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c @@ -1826,7 +1826,7 @@ show_import(nvlist_t *config) } if (msgid != NULL) - (void) printf(gettext(" see: http://illumos.org/msg/%s\n"), + (void) printf(gettext(" see: https://illumos.org/msg/%s\n"), msgid); (void) printf(gettext(" config:\n\n")); @@ -4124,7 +4124,7 @@ print_dedup_stats(nvlist_t *config) * pool: tank * status: DEGRADED * reason: One or more devices ... - * see: http://illumos.org/msg/ZFS-xxxx-01 + * see: https://illumos.org/msg/ZFS-xxxx-01 * config: * mirror DEGRADED * c1t0d0 OK @@ -4381,7 +4381,7 @@ status_callback(zpool_handle_t *zhp, void *data) } if (msgid != NULL) - (void) printf(gettext(" see: http://illumos.org/msg/%s\n"), + (void) printf(gettext(" see: https://illumos.org/msg/%s\n"), msgid); if (config != NULL) { diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c index 54cb224a7cdd..0ef9065de044 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c @@ -2631,7 +2631,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, cmn_err(CE_WARN, "pool '%s' could not be " "loaded as it was last accessed by " "another system (host: %s hostid: 0x%lx). " - "See: http://illumos.org/msg/ZFS-8000-EY", + "See: https://illumos.org/msg/ZFS-8000-EY", spa_name(spa), hostname, (unsigned long)hostid); return (SET_ERROR(EBADF)); -- 2.11.0 From d978609abb92e2e01a33f2dcd8ea2a469ebc3e5c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 29 Apr 2016 19:38:53 +0200 Subject: [PATCH 244/255] ZFS: Add a (tunable) lower limit for the dynamic write buffer Default to using 0.5% of the system's physical memory. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 13 +++++++++++++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 1 + sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c | 5 +++++ .../contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h | 1 + 4 files changed, 20 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 9c844e30e9e5..61b1bcdffef2 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -6276,6 +6276,19 @@ arc_init(void) zfs_dirty_data_max = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); } + /* + * If memory is tight, reduce the dynamic write buffer + * as low as 0.5% of the physical memory. This allows + * a system with 1 GB of physical memory to remain stable + * while building kernels in a loop for days. + * Using 1% (10 MB) proved to be too much. + * + * Systems with 2 GB of physical memory don't seem to + * reach the lower limit while building kernels. + */ + if (zfs_dirty_data_max_min == 0) { + zfs_dirty_data_max_min = ptob(physmem) / 200; + } #ifdef _KERNEL if (TUNABLE_INT_FETCH("vfs.zfs.prefetch_disable", &zfs_prefetch_disable)) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 51ded24b981f..7c1e3a6df36a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1394,6 +1394,7 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) last_max = zfs_dirty_data_max_internal / (1024 * 1024); DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); } + zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, zfs_dirty_data_max_min); if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { zfs_dynamic_write_buffer_hits++; } diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index d99323fd21a3..b8d742844e37 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -105,6 +105,7 @@ uint64_t zfs_dirty_data_max; uint64_t zfs_dirty_data_max_internal; uint64_t zfs_dirty_data_max_max = 4ULL * 1024 * 1024 * 1024; +uint64_t zfs_dirty_data_max_min = 0; int zfs_dirty_data_max_percent = 10; /* @@ -154,6 +155,10 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_internal, CTLFLAG_RD, &zfs_dirty_data_max_internal, 0, "The currently active limit of dirty data (may dynamically change)"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_min, CTLFLAG_RWTUN, + &zfs_dirty_data_max_min, 0, + "The absolute lower cap on dirty_data_max_internal when auto calculating"); + static int sysctl_zfs_dirty_data_max_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs, OID_AUTO, dirty_data_max_percent, CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h index ee97b5719cd2..c0444da17104 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h @@ -52,6 +52,7 @@ struct dsl_scan; extern uint64_t zfs_dirty_data_max; extern uint64_t zfs_dirty_data_max_internal; extern uint64_t zfs_dirty_data_max_max; +extern uint64_t zfs_dirty_data_max_min; extern uint64_t zfs_dirty_data_sync; extern int zfs_dirty_data_max_percent; extern int zfs_delay_min_dirty_percent; -- 2.11.0 From e0e622616a803f9537901053dd9cd2c3669a0d1e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 3 May 2016 16:13:29 +0200 Subject: [PATCH 245/255] dmu_tx_assign(): Limit zfs_dirty_data_max_internal growth ... to steps of zfs_dirty_data_max_min or less. As dmu_tx_assign() is called quite frequently the growth should probably additionally be delayed based on time, though. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 7c1e3a6df36a..6342d397928a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1346,7 +1346,7 @@ int dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) { int err; - static uint64_t last_max; + static uint64_t last_max = 4ULL * 1024 * 1024 * 1024; ASSERT(tx->tx_txg == 0); ASSERT(txg_how == TXG_WAIT || txg_how == TXG_NOWAIT || @@ -1390,9 +1390,10 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count / 2)); } - if (last_max != (zfs_dirty_data_max_internal / (1024 * 1024))) { - last_max = zfs_dirty_data_max_internal / (1024 * 1024); - DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); + zfs_dirty_data_max_internal = MIN(last_max + zfs_dirty_data_max_min, zfs_dirty_data_max_internal); + if (last_max != zfs_dirty_data_max_internal) { + last_max = zfs_dirty_data_max_internal; + DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max / (1024 * 1024)); } zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, zfs_dirty_data_max_min); if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { -- 2.11.0 From d9fb88602a9d936e77d19a0a59f652fb1fd936e9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 8 May 2016 14:13:41 +0200 Subject: [PATCH 246/255] dmu_tx_assign(): Use local variable and use the whole v_free_count if it's below the target Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 6342d397928a..0c10b307698b 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1382,27 +1382,29 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) * honor that as a hard cap so it remains a usable tunable value. */ if (zio_use_uma && zfs_dynamic_write_buffer) { - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); + uint64_t new_max; + + new_max = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); if (vm_cnt.v_free_count > vm_cnt.v_free_target) { - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, + new_max = MIN(new_max, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); } else { - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, - ptob(vm_cnt.v_free_count / 2)); + new_max = MIN(new_max, vm_cnt.v_free_count); } - zfs_dirty_data_max_internal = MIN(last_max + zfs_dirty_data_max_min, zfs_dirty_data_max_internal); - if (last_max != zfs_dirty_data_max_internal) { - last_max = zfs_dirty_data_max_internal; + new_max = MIN(last_max + zfs_dirty_data_max_min, new_max); + if (last_max != new_max) { + last_max = new_max; DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max / (1024 * 1024)); } - zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, zfs_dirty_data_max_min); - if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { + new_max = MAX(new_max, zfs_dirty_data_max_min); + if (new_max < zfs_dirty_data_max) { zfs_dynamic_write_buffer_hits++; } if (zfs_lowest_dynamic_write_buffer_limit == 0 || - zfs_lowest_dynamic_write_buffer_limit >= zfs_dirty_data_max_internal) { - zfs_lowest_dynamic_write_buffer_limit = zfs_dirty_data_max_internal; + zfs_lowest_dynamic_write_buffer_limit >= new_max) { + zfs_lowest_dynamic_write_buffer_limit = new_max; } + zfs_dirty_data_max_internal = new_max; } else { zfs_dirty_data_max_internal = zfs_dirty_data_max; } -- 2.11.0 From 9250219e3fb67d67e0cf1bdbd3fcd0943641ac30 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 16:05:49 +0200 Subject: [PATCH 247/255] sys/vm: Limit the inactive pages more aggressively Currently the ZFS ARC does not take the inactive pages into account when calculating its target size. If there's no limit for the inactive pages, the ARC may shrink to its own limit while the number of inactive pages continues to grow: last pid: 28429; load averages: 0.48, 0.46, 0.41 up 0+03:39:07 17:24:59 91 processes: 2 running, 88 sleeping, 1 waiting CPU: 1.4% user, 0.0% nice, 12.7% system, 0.2% interrupt, 85.7% idle Mem: 396M Active, 489M Inact, 986M Wired, 292K Cache, 5202K Buf, 43M Free ARC: 351M Total, 90M MFU, 44M MRU, 6839K Anon, 7810K Header, 203M Other, 350M Target Swap: 2048M Total, 99M Used, 1949M Free, 4% Inuse PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND 11 root 2 155 ki31 0K 32K RUN 0 377:37 170.34% idle 26625 fk 17 36 0 175M 24504K uwait 1 0:09 8.40% git 0 root 468 -16 0 0K 7488K swapin 1 3:29 6.26% kernel 22 root 1 20 - 0K 16K geli:w 1 4:16 5.06% g_eli[1] ada0s1d [...] 2015 Sep 21 17:24:58: Scan goals in the previous minute: Update active LRU/deactivate pages 60 2015 Sep 21 17:24:58: Seconds since last 'Move inactive to cache or free' pass: 1477 2015 Sep 21 17:24:58: Seconds since last 'Launder dirty pages' pass: 9273 With this commit, the system lets the ARC indirectly put pressure on the inactive pages until a given target is reached. A couple of sysctls can be used to set various limits, the auto-tuned default should work reasonably well, though. Note that suboptimal tuning can result in excessive paging. Screenshot (made with previous version of this commit): https://www.fabiankeil.de/bilder/electrobsd/kernel-compilation-with-inactive-page-limit-enabled.png XXX: After rebasing on r300865 this commit caused a bunch of conflicts that may not have been addressed ideally. Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 194 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 157 insertions(+), 37 deletions(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 3e36f7c1c3fa..141c087becfa 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -140,15 +140,17 @@ SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, SDT_PROVIDER_DEFINE(vm); SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan); -SDT_PROBE_DEFINE4(vm, , , before__inactive__scan, "struct vm_domain *vmd", - "int pass", "int page_shortage", "int deficit"); -SDT_PROBE_DEFINE5(vm, , , after__inactive__scan, "struct vm_domain *vmd", +SDT_PROBE_DEFINE5(vm, , , before__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int deficit", "int inactive_page_surplus"); +SDT_PROBE_DEFINE6(vm, , , after__inactive__scan, "struct vm_domain *vmd", "int pass", "int page_shortage", "int addl_page_shortage", - "int vnodes_skipped"); + "int vnodes_skipped", "int inactive_page_surplus"); SDT_PROBE_DEFINE3(vm, , , before__active__scan, "struct vm_domain *vmd", "int pass", "int page_shortage"); SDT_PROBE_DEFINE3(vm, , , after__active__scan, "struct vm_domain *vmd", "int pass", "int page_shortage"); +SDT_PROBE_DEFINE3(vm, , , checked__inactive__pages, "int pages_to_free", + "int pages_above_limit", "int enforced_limit"); #if !defined(NO_SWAPPING) /* the kernel process "vm_daemon"*/ @@ -230,6 +232,36 @@ SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts, SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts, CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages"); +static int inactive_page_limit_enabled = 1; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_enabled, CTLFLAG_RW, + &inactive_page_limit_enabled, 0, + "Free inactive pages above the target more aggressively. " + "Values: 0 (disabled), 1 (soft mode, only apply limit if free " + "page count is low), 2 (ignore free count)"); + +static int inactive_page_limit_offset_i; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_offset_i, CTLFLAG_RW, + &inactive_page_limit_offset_i, 0, + "Number of inactive pages relative to the inactive target " + "required for inactive pages to be freed."); + +static int inactive_page_limit_offset_f; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_offset_f, CTLFLAG_RW, + &inactive_page_limit_offset_f, 0, + "Number of free pages relative to the free target required for " + "the inactive memory limit to be applied."); + +static int inactive_pages_to_free_max = 1000; +SYSCTL_INT(_vm, OID_AUTO, inactive_pages_to_free_max, CTLFLAG_RW, + &inactive_pages_to_free_max, 0, + "Maximum number of inactive pages above the target to free at once."); + +static int inactive_page_limit_threshold = 1000; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_threshold, CTLFLAG_RW, + &inactive_page_limit_threshold, 0, + "Number of inactive pages above the limit required " + "to trigger a inactive page reduction."); + static int pageout_lock_miss; SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss, CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout"); @@ -848,6 +880,41 @@ unlock_mp: return (error); } +static int +vm_pageout_get_inactive_page_surplus(void) +{ + int pages_to_free; + int pages_above_limit; + int enforced_limit; + + /* Return early so the DTrace probe does not fire. */ + if (!inactive_page_limit_enabled) + return (0); + + enforced_limit = vm_cnt.v_inactive_target + inactive_page_limit_offset_i; + pages_above_limit = vm_cnt.v_inactive_count - enforced_limit; + + /* + * We want to free inactive pages if the threshold of inactive + * pages above the limit is reached and we are either using + * a hard limit, or the number of free pages is below the + * free page limit. + */ + if ((pages_above_limit >= inactive_page_limit_threshold) && + ((inactive_page_limit_enabled == 2) || + (vm_paging_target() + inactive_page_limit_offset_f > 0))) { + pages_to_free = imin(inactive_pages_to_free_max, + pages_above_limit); + } else { + pages_to_free = 0; + } + + SDT_PROBE3(vm, , , checked__inactive__pages, pages_to_free, + pages_above_limit, enforced_limit); + + return (pages_to_free); +} + #define VMD_PASS_MAX 3 /* * vm_pageout_scan does the dirty work for the pageout daemon. @@ -870,15 +937,35 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) int maxlaunder, maxscan, page_shortage, scan_tick, scanned; int starting_page_shortage, vnodes_skipped; boolean_t pageout_ok, queue_locked; + int inactive_page_surplus; KASSERT(pass <= VMD_PASS_MAX, ("vm_pageout_scan: Invalid pass code %d", pass)); /* + * The addl_page_shortage is the number of temporarily + * stuck pages in the inactive queue. In other words, the + * number of pages from the inactive count that should be + * discounted in setting the target for the active queue scan. + */ + addl_page_shortage = 0; + + /* + * Calculate the number of pages that we want to free. + */ + if (pass > 0) { + deficit = atomic_readandclear_int(&vm_pageout_deficit); + page_shortage = vm_paging_target() + deficit; + inactive_page_surplus = vm_pageout_get_inactive_page_surplus(); + } else + page_shortage = deficit = inactive_page_surplus = 0; + starting_page_shortage = page_shortage; + + /* * If we need to reclaim memory ask kernel caches to return * some. We rate limit to avoid thrashing. */ - if (vmd == &vm_dom[0] && pass > 0 && + if (vmd == &vm_dom[0] && pass > 0 && page_shortage > 0 && (time_uptime - lowmem_uptime) >= lowmem_period) { /* * Decrease registered cache sizes. @@ -894,26 +981,6 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) } /* - * The addl_page_shortage is the number of temporarily - * stuck pages in the inactive queue. In other words, the - * number of pages from the inactive count that should be - * discounted in setting the target for the active queue scan. - */ - addl_page_shortage = 0; - - /* - * Calculate the number of pages that we want to free. This number - * can be negative if many pages are freed between the wakeup call to - * the page daemon and this calculation. - */ - if (pass > 0) { - deficit = atomic_readandclear_int(&vm_pageout_deficit); - page_shortage = vm_paging_target() + deficit; - } else - page_shortage = deficit = 0; - starting_page_shortage = page_shortage; - - /* * maxlaunder limits the number of dirty pages we flush per scan. * For most systems a smaller value (16 or 32) is more robust under * extreme memory and disk pressure because any unnecessary writes @@ -928,10 +995,18 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) if (pass > 1) maxlaunder = 10000; + /* + * Prevent laundering if there's no page shortage and we are + * merely trying to free inactive pages. Otherwise we may end + * up swapping before it's really necessary. + */ + if (page_shortage <= 0) + maxlaunder = 0; + vnodes_skipped = 0; - SDT_PROBE4(vm, , , before__inactive__scan, vmd, pass, page_shortage, - deficit); + SDT_PROBE5(vm, , , before__inactive__scan, vmd, pass, page_shortage, + deficit, inactive_page_surplus); /* * Start scanning the inactive queue for pages that we can free. The @@ -944,7 +1019,8 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) vm_pagequeue_lock(pq); queue_locked = TRUE; for (m = TAILQ_FIRST(&pq->pq_pl); - m != NULL && maxscan-- > 0 && page_shortage > 0; + m != NULL && maxscan-- > 0 && + (page_shortage > 0 || inactive_page_surplus > 0); m = next) { vm_pagequeue_assert_locked(pq); KASSERT(queue_locked, ("unlocked inactive queue")); @@ -1080,6 +1156,7 @@ free_page: vm_page_free(m); PCPU_INC(cnt.v_dfree); --page_shortage; + --inactive_page_surplus; } else if ((object->flags & OBJ_DEAD) != 0) { /* * Leave dirty pages from dead objects at the front of @@ -1107,13 +1184,19 @@ requeue_page: vm_pagequeue_lock(pq); queue_locked = TRUE; vm_page_requeue_locked(m); - } else if (maxlaunder > 0) { + } else if (maxlaunder > 0 && page_shortage > 0) { /* - * We always want to try to flush some dirty pages if - * we encounter them, to keep the system stable. + * As long as there is a page shortage, we try to + * flush some dirty pages if we encounter them, to + * keep the system stable. * Normally this number is small, but under extreme * pressure where there are insufficient clean pages * on the inactive queue, we may have to go all out. + * + * XXX: We probably should not bother laundering + * until we know that there might be a chance + * that we will not be able to free the required + * amount of pages to take care of the page shortage. */ if (object->type != OBJT_SWAP && @@ -1158,15 +1241,27 @@ relock_queue: } vm_pagequeue_unlock(pq); - SDT_PROBE5(vm, , , after__inactive__scan, vmd, pass, page_shortage, - addl_page_shortage, vnodes_skipped); + /* + * If the page shortage has been taken care of, or if we were + * just trying to free surplus inactive pages, the locked pages + * are more or less meaningless. Reset the counter to prevent + * pointless swapping. + */ + if (page_shortage <= 0) + addl_page_shortage = 0; + + SDT_PROBE6(vm, , , after__inactive__scan, vmd, pass, page_shortage, + addl_page_shortage, vnodes_skipped, inactive_page_surplus); #if !defined(NO_SWAPPING) /* - * Wakeup the swapout daemon if we didn't free the targeted number of - * pages. + * Wakeup the swapout daemon if we didn't free the targeted number + * of pages and we are either desperate or there are no inactive + * pages to free left (in which case we will be desperate soon + * enough). */ - if (vm_swap_enabled && page_shortage > 0) + if (vm_swap_enabled && page_shortage > 0 && + (pass > 1 || !vm_pageout_get_inactive_page_surplus())) vm_req_vmdaemon(VM_SWAP_NORMAL); #endif @@ -1587,7 +1682,9 @@ vm_pageout_worker(void *arg) /* * Might the page daemon receive a wakeup call? */ - if (vm_pageout_wanted) { + /* XXX: After r300865 this may no longer work. Investigate! */ + if (vm_pageout_wanted || + vm_pageout_get_inactive_page_surplus() > 0) { /* * No. Either vm_pageout_wanted was set by another * thread during the previous scan, which must have @@ -1656,6 +1753,29 @@ vm_pageout_init(void) vm_cnt.v_inactive_target = vm_cnt.v_free_count / 3; /* + * Apply the inactive memory limit before ZFS's + * dirty data limit kicks in. + */ + inactive_page_limit_offset_f = vm_cnt.v_page_count / 9 + - vm_cnt.v_free_target; + + /* + * A man carrying a large stone through the savanne was asked why + * he would do such a strange thing. His explanation: "If a lion + * comes, I'll through away the stone which will allow me to run + * faster." + * + * For similar reasons the inactive page limit defaults to + * allowing twice the number of inactive pages the vm targets + * itself. + * + * If sudden memory pressure comes, the inactive page reserve can + * be thrown away to make it less likely that the system has to + * start paging. + */ + inactive_page_limit_offset_i = vm_cnt.v_inactive_target; + + /* * Set the default wakeup threshold to be 10% above the minimum * page limit. This keeps the steady state out of shortfall. */ -- 2.11.0 From 2370c182b4d291d8a0fb68e5c069fc45e05d3c25 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 27 May 2016 11:56:18 +0200 Subject: [PATCH 248/255] ZFS: Default to reducing the dirty data buffer to 50% of the maximum It's currently unclear whether or not dynamically scaling the buffer is a good idea at all. Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 61b1bcdffef2..477a528e2411 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -6277,17 +6277,23 @@ arc_init(void) zfs_dirty_data_max_max); } /* - * If memory is tight, reduce the dynamic write buffer - * as low as 0.5% of the physical memory. This allows - * a system with 1 GB of physical memory to remain stable - * while building kernels in a loop for days. - * Using 1% (10 MB) proved to be too much. + * Calculate minimum amount of dirty data allowed per pool. * - * Systems with 2 GB of physical memory don't seem to - * reach the lower limit while building kernels. + * Initially the default was 0.5% of the system's memory + * and was supposed to prevent crashes when reproducing + * ElectroBSD on systems with 1 GB or less. + * + * While it made the crashes less common, it did not actually + * prevent them. + * + * The problem was eventually tracked down to a geli bug + * and fixed (#209759), therefore it's unclear if dynamically + * reducing the amount of dirty data allowed is still useful. + * + * For now it's set to 50% of the maximum. */ if (zfs_dirty_data_max_min == 0) { - zfs_dirty_data_max_min = ptob(physmem) / 200; + zfs_dirty_data_max_min = zfs_dirty_data_max / 2; } #ifdef _KERNEL -- 2.11.0 From b64bced8e8da9853242dbd9e20826ff9aa38a991 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 23 Oct 2015 23:36:28 +0200 Subject: [PATCH 249/255] zfs: Do not advertise edonr which isn't supported yet illumos 4185 ("add new cryptographic checksums to ZFS: SHA-512, Skein, Edon-R") was intentionally merged only partially in r289422, without adding support for skein, sha512 and edonr on FreeBSD. Support for skein and sha512 was added later on, but edonr remains missing. Without this commit, zfs(8) correctly rejects edonr, the error message claims that it is supported, though: fk@r500 ~ $zfs set checksum=edonr tank cannot set property for 'tank': 'checksum' must be one of 'on | off | fletcher2 | fletcher4 | sha256 | sha512 | skein | edonr' Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=204055 PR submission date: 2015-10-27 (before sha512 and skein were added) PR update date: Not yet updated as upstream does not seem to care anyway. --- sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c index 079d86043f4d..2a4b18abb2a0 100644 --- a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c +++ b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c @@ -242,11 +242,11 @@ zfs_prop_init(void) ZIO_CHECKSUM_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "on | off | fletcher2 | fletcher4 | sha256 | sha512 | " - "skein | edonr", "CHECKSUM", checksum_table); + "skein", "CHECKSUM", checksum_table); zprop_register_index(ZFS_PROP_DEDUP, "dedup", ZIO_CHECKSUM_OFF, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "on | off | verify | sha256[,verify], sha512[,verify], " - "skein[,verify], edonr,verify", "DEDUP", dedup_table); + "skein[,verify]", "DEDUP", dedup_table); zprop_register_index(ZFS_PROP_COMPRESSION, "compression", ZIO_COMPRESS_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, -- 2.11.0 From 521c51ddb2c10abdd303f4a968fafc1c55f6029b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 29 May 2015 10:46:06 +0200 Subject: [PATCH 250/255] Import cloudiatr 2016-11-02-89a8898 Obtained from: ElectroBSD --- usr.sbin/Makefile | 1 + usr.sbin/cloudiatr/Makefile | 3 + usr.sbin/cloudiatr/cloudiatr | 1275 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1279 insertions(+) create mode 100644 usr.sbin/cloudiatr/Makefile create mode 100755 usr.sbin/cloudiatr/cloudiatr diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index 3d146d6e7704..1b739878676d 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -8,6 +8,7 @@ SUBDIR= adduser \ binmiscctl \ bsdconfig \ camdd \ + cloudiatr \ cdcontrol \ chkgrp \ chown \ diff --git a/usr.sbin/cloudiatr/Makefile b/usr.sbin/cloudiatr/Makefile new file mode 100644 index 000000000000..34ada8582ecd --- /dev/null +++ b/usr.sbin/cloudiatr/Makefile @@ -0,0 +1,3 @@ +SCRIPTS= cloudiatr + +.include diff --git a/usr.sbin/cloudiatr/cloudiatr b/usr.sbin/cloudiatr/cloudiatr new file mode 100755 index 000000000000..461815d7e9c3 --- /dev/null +++ b/usr.sbin/cloudiatr/cloudiatr @@ -0,0 +1,1275 @@ +#!/bin/sh + +########################################################################### +# cloudiatr +# +# Buzzword-compliant remote OS eviction tool. For details see: +# https://www.fabiankeil.de/gehacktes/cloudiatr/ +# +# Copyright (c) 2014-2016 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# It's important that this function is called before any other +# function except cloudiatr_main(), otherwise fatal errors may +# not be caught. +cloudiatr_init() { + local mode="${1}" + + set -e + cloudiatr_init_globals + + cloudiatr_load_config_file "${CLOUDIATR_CONFIG_FILE}" + + cloudiatr_check_config +} + +cloudiatr_fyi() { + local message="${*}" + + echo "cloudiatr: $message" +} + +cloudiatr_wtf() { + local complaints="${*}" + if [ -z "${complaints}" ]; then + complaints="cloudiatr_wtf(): No complaints?" + fi + cloudiatr_fyi "${complaints}" 1>&2 + return 1 +} + +cloudiatr_check_config() { + local v_flag \ + mandatory_variable optional_variable value fail + + v_flag="${1}" + fail=0 + + for mandatory_variable in ${CLOUDIA_MANDATORY_VARIABLES}; do + value="$(eval 'echo $'"${mandatory_variable}")" + if [ -z "${value}" ]; then + cloudiatr_wtf "Fatal error: ${mandatory_variable} is unset" + fail=1 + elif [ "${v_flag}" = "-v" ]; then + echo "${mandatory_variable}='${value}'" + fi + done + if [ "${v_flag}" = "-v" ]; then + for optional_variable in ${CLOUDIA_OPTIONAL_VARIABLES}; do + value="$(eval 'echo $'"${optional_variable}")" + echo "${optional_variable}='${value}'" + done + fi + return $fail +} + +cloudiatr_show_config() { + cloudiatr_check_config -v +} + +cloudiatr_load_config_file() { + local config_file="${1}" + + if [ -f "${config_file}" ]; then + . "${config_file}" + return 0 + fi + cloudiatr_wtf "Config file ${config_file} does not exist. You can use '$0 -f path/to/file ...' to specify a different one" + return 1 +} + +cloudiatr_init_globals() { + + CLOUDIATR_VERSION="2016-11-02-89a8898" + + CLOUDIATR_NEW_SYSTEM_DIR=/cloudiatr + # Only needs to be enough for a stripped-down bootfs + CLOUDIATR_BPOOL_PARTITION_SIZE=200M + # Has to be enough for the rest of the OS including the "permanent" + # /boot that is only used to (re)populate the bootfs on the bpool. + CLOUDIATR_RPOOL_PARTITION_SIZE=4G + CLOUDIATR_SWAP_PARTITION_SIZE=4G + + # Set to true to use the existing partition layout. + # + # Only expected to work if the layout was created by a previous + # cloudiatr run. By default the partitions 2, 3 and 4 will be + # overwritten and partition 1 is expected to contain working bootcode. + CLOUDIATR_REUSE_GPART_SETUP=false + + # Usually changing the partition numbers usually is not necessary + # and changing them after the installation is likely to result in + # data loss. + CLOUDIATR_BOOTCODE_PARTITION=1 + CLOUDIATR_BPOOL_PARTITION=2 + CLOUDIATR_RPOOL_PARTITION=3 + CLOUDIATR_SWAP_PARTITION=4 + CLOUDIATR_DPOOL_PARTITION=5 + + CLOUDIATR_BPOOL_NAME="bpool" + CLOUDIATR_RPOOL_NAME="rpool" + + # Note that DEFAULT is a fallback documented in rc.conf(5). + # Not changing this variable to the actual network interface + # is likely to cause problems if there are more than one nics + # and you aren't using DHCP for all of them. + CLOUDIATR_NIC="DEFAULT" + + # Will be created + CLOUDIATR_RPOOL_KEY_NAME="${CLOUDIATR_RPOOL_NAME}.key" + CLOUDIATR_RPOOL_KEY="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_KEY_NAME}" + + # If CLOUDIATR_DIST_IMAGE is set, an image containing the dist tarballs + # has to be put in place by the user before the eviction. If it's unset, + # CLOUDIATR_DIST_DIR has to be populated before cloudiatr is executed. + CLOUDIATR_DIST_IMAGE="" + CLOUDIATR_DIST_IMAGE_SHA256="" + CLOUDIATR_DIST_DIR="/usr/electrobsd-dist/" + + # Whether or not the distribution tarballs should be copied to the + # newly installed system (for example to reuse them when setting + # up jails). + CLOUDIATR_SAVE_DIST_DIR="false" + + # Default to using all the detected ada(4) devices + CLOUDIATR_DISKS="$(cloudiatr_autodetect_disks)" + + # Changing these should only be necessary if there's more + # than one disk and you don't want to a mirror. + CLOUDIATR_BPOOL_LAYOUT="default" + CLOUDIATR_RPOOL_LAYOUT="default" + + CLOUDIATR_GELI_KEY_LENGTH=256 + CLOUDIATR_GELI_EALGO=AES-XTS + + CLOUDIATR_SSHD_HOST_KEY_ALGORITHMS="rsa ecdsa ed25519" + + # Set to 'true' to ingore some safety-checks and increase the potential damage. + # Includes "geli kill -a" which is not limited to the disks specified above. + # Do not enable this unless the system that is being evicted doesn't contain + # any data you care about. + CLOUDIATR_MURDER_DEATH_KILL_REQUESTED=false + + # Set to 'true' to skip the image checksum check. + # "It may be insecure, but look how fast it is!" + CLOUDIATR_CHECKSUM_SMECKSUM=false + + # The config file is sourced and may overwrite any of the values above + # and most functions in this file (zogftw-style). + CLOUDIATR_CONFIG_FILE="${CLOUDIATR_CONFIG_FILE=/etc/cloudiatr.conf}" + + # A user that should be created and allowed to "su" on the new system. + CLOUDIATR_NEW_USER="cloudiatr" + + # This password is used for both root and CLOUDIATR_NEW_USER. + # + # Note that the created system will not accept root logins through + # ssh (FreeBSD default). CLOUDIATR_NEW_USER may use ssh, but has + # to use public key authentication. + CLOUDIATR_INITIAL_PASSWORD="${CLOUDIATR_NEW_USER}" + + # Local timezone. For details see tzsetup(8). + CLOUDIATR_TIME_ZONE="Europe/Berlin" + + # When set to true, cloudiatr will execute ntpdate at installtime. + # The server(s) being used depend on the install distfiles. + CLOUDIATR_USE_NTPDATE="false" + + # Any alignment should work, 1M is often recommended to prevent + # write-amplification which can result in performance degradation. + # It can also be advantageous for trimming SSDs. + CLOUDIATR_GPART_ALIGNMENT="1M" + + # Optional keyboard map for the virtual console. + # For details see kbdmap. + CLOUDIATR_KBDMAP="de.kbd" + + # Additional distributions to extract. Example: src, lib32 + CLOUDIATR_EXTRA_DISTRIBUTIONS="" + + # Value for rc.conf's rether_enable entry which controls + # whether or not MAC addresses are randomized (on ElectroBSD). + CLOUDIATR_RETHER_ENABLE="NO" + + # Set to true to not bother the user about with questions. + : "${CLOUDIATR_DONT_ASK_JUST_KISS=false}" + + # Set to true (default) to use the added swap partitions right + # after creating them. This allows installations on systems that + # have insufficient memory (512 MB, for example) and no previously + # configured swap devices. + # + # While this option is not expected to cause problems, if you + # are absolutely sure that enough memory is available you can + # disable the behaviour by setting the variable to "false". + CLOUDIATR_USE_SWAP_WHILE_INSTALLING="true" + + # Apply a workaround that is required to boot on + # Lenovo laptops like the T520 + CLOUDIATR_APPLY_LENOVO_WORKAROUND="false" + + # If these variables aren't set to some value, cloudiatr will abort. + # Sane values are a good idea but not mandatory. + CLOUDIA_MANDATORY_VARIABLES="\ + CLOUDIATR_APPLY_LENOVO_WORKAROUND \ + CLOUDIATR_BPOOL_LAYOUT \ + CLOUDIATR_BPOOL_NAME \ + CLOUDIATR_BPOOL_PARTITION_SIZE \ + CLOUDIATR_CHECKSUM_SMECKSUM \ + CLOUDIATR_CONFIG_FILE \ + CLOUDIATR_DISKS \ + CLOUDIATR_DIST_DIR \ + CLOUDIATR_DONT_ASK_JUST_KISS \ + CLOUDIATR_GELI_KEY_LENGTH \ + CLOUDIATR_GELI_EALGO \ + CLOUDIATR_GPART_ALIGNMENT \ + CLOUDIATR_HOSTNAME \ + CLOUDIATR_INITIAL_PASSWORD \ + CLOUDIATR_MURDER_DEATH_KILL_REQUESTED \ + CLOUDIATR_NEW_SYSTEM_DIR \ + CLOUDIATR_NEW_USER \ + CLOUDIATR_NIC \ + CLOUDIATR_USE_SWAP_WHILE_INSTALLING \ + CLOUDIATR_RETHER_ENABLE \ + CLOUDIATR_REUSE_GPART_SETUP \ + CLOUDIATR_RPOOL_KEY \ + CLOUDIATR_RPOOL_KEY_NAME \ + CLOUDIATR_RPOOL_LAYOUT \ + CLOUDIATR_RPOOL_NAME \ + CLOUDIATR_RPOOL_PARTITION_SIZE \ + CLOUDIATR_SAVE_DIST_DIR \ + CLOUDIATR_SWAP_PARTITION_SIZE \ + CLOUDIATR_TIME_ZONE \ + CLOUDIATR_USE_NTPDATE \ + CLOUDIATR_VERSION \ + " + + # These variables are allowed to be unset + CLOUDIA_OPTIONAL_VARIABLES="\ + CLOUDIATR_DEFAULTROUTER \ + CLOUDIATR_DIST_IMAGE \ + CLOUDIATR_DIST_IMAGE_SHA256 \ + CLOUDIATR_EXTRA_DISTRIBUTIONS \ + CLOUDIATR_IP_ADDRESS \ + CLOUDIATR_KBDMAP \ + CLOUDIATR_NETMASK \ + " +} + +# Apply the workaround described at: +# https://lists.freebsd.org/pipermail/freebsd-i386/2013-March/010437.html +cloudiatr_apply_lenovo_workaround() { + local disk="${1}" \ + slice_table_original slice_table_new \ + partion_spec cylinders sectors_per_track cylinders_new heads + + if [ -z "${disk}" ]; then + cloudiatr_wtf "cloudiatr_apply_lenovo_workaround: No disk provided" + return 1 + fi + + slice_table_original=$(mktemp -t cloudiatr_slice_table_original) || return 1 + slice_table_new=$(mktemp -t cloudiatr_slice_table_new) || return 1 + + fdisk -p "${disk}" > "${slice_table_original}" || return 1 + + cylinders="$(grep '^g c' "${slice_table_original}" | cut -w -f 2 | cut -d c -f 2)" || return 1 + heads="$(grep '^g c' "${slice_table_original}" | cut -w -f 3 | cut -d h -f 2)" || return 1 + sectors_per_track="$(grep '^g c' "${slice_table_original}" | cut -w -f 4 | cut -d s -f 2)" || return 1 + + cylinders_new="$(expr "${cylinders}" \* "${sectors_per_track}")" || return 1 + + partition_spec="$(tail -n 1 ${slice_table_original})" || return 1 + + echo "# ${disk}" > "${slice_table_new}" || return 1 + echo "g c${cylinders_new} h${heads} s1" >> "${slice_table_new}" || return 1 + echo "${partition_spec}" | sed -e 's@^p 1 0xee@p 1 0x00@' >> "${slice_table_new}" || return 1 + echo "a 1" >> "${slice_table_new}" || return 1 + echo "${partition_spec}" | sed -e 's@^p 1@p 2@' >> "${slice_table_new}" || return 1 + + cloudiatr_fyi "Applying Lenovo workaround:" + diff -u "${slice_table_original}" "${slice_table_new}" || true + fdisk -f "${slice_table_new}" "${disk}" || return 1 + rm "${slice_table_original}" "${slice_table_new}" || return 1 +} + +cloudiatr_gpart_disk() { + local disk="${1}" \ + disk_name + + disk_name="${disk##*/}" + + gpart create -s gpt "${disk}" || return 1 + + gpart add -s 512 -t freebsd-boot \ + -i "${CLOUDIATR_BOOTCODE_PARTITION}" "${disk}" || return 1 + gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot \ + -i "${CLOUDIATR_BOOTCODE_PARTITION}" "${disk}" || return 1 + + gpart add -s "${CLOUDIATR_BPOOL_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "${CLOUDIATR_BPOOL_NAME}-${disk_name}" -t freebsd-zfs \ + -i "${CLOUDIATR_BPOOL_PARTITION}" "${disk}" || return 1 + gpart add -s "${CLOUDIATR_RPOOL_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "${CLOUDIATR_RPOOL_NAME}-${disk_name}" -t freebsd-zfs \ + -i "${CLOUDIATR_RPOOL_PARTITION}" "${disk}" || return 1 + gpart add -s "${CLOUDIATR_SWAP_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "swap-${disk_name}" -t freebsd-swap \ + -i "${CLOUDIATR_SWAP_PARTITION}" "${disk}" || return 1 + + # Reserve what's left for the data pool + gpart add -l "dpool-${disk_name}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -t freebsd-zfs -i "${CLOUDIATR_DPOOL_PARTITION}" "${disk}" || return 1 + + if "${CLOUDIATR_APPLY_LENOVO_WORKAROUND}"; then + cloudiatr_apply_lenovo_workaround "${disk}" || return 1 + fi +} + +cloudiatr_gpart_setup() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Cleaning partition tables (if there are any) ..." + for d in $disks; do + gpart destroy -F "${d}" 2>/dev/null || true + done + + cloudiatr_fyi "Partitioning disks ..." + for d in $disks; do + cloudiatr_gpart_disk "${d}" || return 1 + done +} + +# Use the swap partitions on the given disks while cloudiatr is running. +# This allows to install on a system with 512MB RAM or less and no swap space. +cloudiatr_enable_swap() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Using created swap space while installing ..." + for d in $disks; do + geli onetime -d "${d}p${CLOUDIATR_SWAP_PARTITION}" || return 1 + swapon "${d}p${CLOUDIATR_SWAP_PARTITION}.eli" || return 1 + done +} + +cloudiatr_disable_swap() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Trying to disable previously added swap space ..." + for d in $disks; do + swapoff "${d}p${CLOUDIATR_SWAP_PARTITION}.eli" || return 1 + done +} + +cloudiatr_get_geoms() { + local postfix="${1}" \ + d geoms + + for d in ${CLOUDIATR_DISKS}; do + geom_partition="${d}${postfix}" + geoms="${geoms} ${geom_partition}" + done + echo "${geoms## }" +} + +cloudiatr_get_gpart_labels() { + local postfix="${1}" + + for d in ${CLOUDIATR_DISKS}; do + d="${d##/dev/}" + gpart show -l -p "${d}" 2>/dev/null | awk '$3 == "'"${d}${postfix}"'" {printf "%s ", $4}' + done + echo +} + +# Depends on geli being already setup +cloudiatr_create_rpool() { + local \ + rpool_elis pool_layout + + rpool_elis="$(cloudiatr_get_geoms "p${CLOUDIATR_RPOOL_PARTITION}.eli")" + pool_layout="${CLOUDIATR_RPOOL_LAYOUT}" + + if [ "${pool_layout}" = "default" ]; then + pool_layout="$(cloudiatr_get_default_pool_layout)" + fi + + cloudiatr_fyi "Creating root pool '${CLOUDIATR_RPOOL_NAME}' on ${rpool_elis}. Pool layout: ${pool_layout}" + + zpool create -o version=28 -o failmode=continue \ + -O compression=lzjb -O checksum=sha256 \ + "${CLOUDIATR_RPOOL_NAME}" ${pool_layout##single-disk} $rpool_elis + + zfs set mountpoint="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}" "${CLOUDIATR_RPOOL_NAME}" + + zfs create "${CLOUDIATR_RPOOL_NAME}/boot" + # We currently use no dedicated dataset for /etc as the kernel expects parts of it + # to be available once the rootfs has been mounted. Having two /etc's can be a bit + # of a hassle on updates and thus doesn't seem like a good default. + #zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/etc" + zfs create "${CLOUDIATR_RPOOL_NAME}/home" + zfs create "${CLOUDIATR_RPOOL_NAME}/home/${CLOUDIATR_NEW_USER}" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/tmp" + zfs create "${CLOUDIATR_RPOOL_NAME}/usr" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/src" + zfs create "${CLOUDIATR_RPOOL_NAME}/var" + zfs create "${CLOUDIATR_RPOOL_NAME}/usr/local" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/local/etc" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/local/src" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports" + zfs create -o compression=off -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports/distfiles" + zfs create -o compression=off -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports/packages" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/crash" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/db" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/db/pkg" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/empty" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/log" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/mail" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/run" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/tmp" + + chmod 0750 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/crash" + chgrp mail "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/mail" + chmod 0775 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/mail" + chmod 0555 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/empty" + chflags schg,nouarch "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/empty" + chmod 1777 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/tmp" + chmod 1777 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/tmp" + + zfs set readonly=on "${CLOUDIATR_RPOOL_NAME}/var/empty" +} + +cloudiatr_kernel_module_is_loaded() { + local module="${1}" + # Can't use 'kldstat -m foo' as it requires a module file on disk. WTF? + kldstat | grep -q "${module}" +} + +cloudiatr_mount_dist_image() { + # intentionally leaks non-local variable md + + if [ ! -f "${CLOUDIATR_DIST_IMAGE}" ]; then + cloudiatr_wtf "File ${CLOUDIATR_DIST_IMAGE} does not exist" + return 1 + fi + + if "${CLOUDIATR_CHECKSUM_SMECKSUM}"; then + cloudiatr_fyi "Checksum smecksum" + else + cloudiatr_fyi "Checking checksum for image file ${CLOUDIATR_DIST_IMAGE} ..." + sha256 -c "${CLOUDIATR_DIST_IMAGE_SHA256}" "${CLOUDIATR_DIST_IMAGE}" + fi + md=$(mdconfig -f "${CLOUDIATR_DIST_IMAGE}") + if [ -z "${md}" ]; then + return 1 + fi + + for potential_partition in "/dev/${md}a" "/dev/${md}p2"; do + if [ -c "${potential_partition}" ]; then + cloudiatr_fyi "Trying to mount ${potential_partition} ..." + mount -o ro "${potential_partition}" /mnt/ || return 1 + cloudiatr_fyi "Mounting ${potential_partition} worked..." + fi + done +} + +cloudiatr_extract_distribution() { + local chroot_dir \ + md extra_distribution + + chroot_dir="${1}" + + if [ -n "${CLOUDIATR_DIST_IMAGE}" ]; then + cloudiatr_mount_dist_image + else + cloudiatr_fyi "No CLOUDIATR_DIST_IMAGE specified." + cloudiatr_fyi "Using CLOUDIATR_DIST_DIR=${CLOUDIATR_DIST_DIR}!" + fi + + cloudiatr_fyi "Extracting base in ${chroot_dir} ..." + # Exclude /var/empty as it's read-only + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/base.txz" --exclude ./var/empty/) + + cloudiatr_fyi "Creating directories that were missing in the base tarball" + chroot "${chroot_dir}" mtree -f /etc/mtree/BSD.root.dist -d -e -u + chroot "${chroot_dir}" mtree -f /etc/mtree/BSD.var.dist -d -e -u -p var + + for extra_distribution in ${CLOUDIATR_EXTRA_DISTRIBUTIONS}; do + cloudiatr_fyi "Extracting extra distribution '${extra_distribution}'" + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/${extra_distribution}.txz") + done + + cloudiatr_fyi "Extracting kernel (without symbols) in ${chroot_dir} ..." + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/kernel.txz" --exclude "*.symbols") + + if [ -n "${CLOUDIATR_DIST_IMAGE}" ]; then + umount /mnt + mdconfig -d -u ${md##md} + fi +} + +cloudiatr_setup_new_user() { + local ssh_dir="${chroot_dir}/home/${CLOUDIATR_NEW_USER}/.ssh" + + cloudiatr_fyi "Creating user '${CLOUDIATR_NEW_USER}'" + + echo "${CLOUDIATR_INITIAL_PASSWORD}" | chroot "${chroot_dir}" \ + pw useradd "${CLOUDIATR_NEW_USER}" -G wheel,operator -h 0 + + # Make sure the user can login through ssh, using the + # authorized_keys file from the installation media. + # + # An extra distribution file may already have created + # the .ssh directory, so don't fail if it already exits. + mkdir -p "${ssh_dir}" + cp -v "${HOME}/.ssh/authorized_keys" "${ssh_dir}" || true + chroot "${chroot_dir}" chown -R "${CLOUDIATR_NEW_USER}" "/home/${CLOUDIATR_NEW_USER}" + chroot "${chroot_dir}" chmod -R go-rwx "/home/${CLOUDIATR_NEW_USER}" +} + +cloudiatr_create_geli_key() { + local keyfile="${1}" + + ( + umask 077 + dd bs=64 count=1 if=/dev/random of="${keyfile}" 2>/dev/null + ) +} + +cloudiatr_setup_geli() { + local disks \ + d + + disks="${*}" + + cloudiatr_create_geli_key "${CLOUDIATR_RPOOL_KEY}" + + mkdir "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups" + + for d in $disks; do + cloudiatr_fyi "Initialising geli on ${d}p${CLOUDIATR_RPOOL_PARTITION} ..." + geli init -b \ + -B "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups/${d##/dev/}p${CLOUDIATR_RPOOL_PARTITION}.eli" \ + -P -K "${CLOUDIATR_RPOOL_KEY}" -l "${CLOUDIATR_GELI_KEY_LENGTH}" \ + -e "${CLOUDIATR_GELI_EALGO}" -s 4096 -V 7 "${d}p${CLOUDIATR_RPOOL_PARTITION}" > /dev/null + done + + cloudiatr_attach_geli_geoms "${CLOUDIATR_RPOOL_KEY}" "p${CLOUDIATR_RPOOL_PARTITION}" +} + +cloudiatr_attach_geli_geoms() { + local keyfile partition_id \ + disk + + keyfile="${1}" + partition_id="${2}" + + for disk in ${CLOUDIATR_DISKS}; do + cloudiatr_fyi "geli-attaching ${disk}${partition_id}" + geli attach -p -k "$keyfile" "${disk}${partition_id}" + done +} + +cloudiatr_detach_geli_geoms() { + local partition_id + + partition_id="${1}" + + for disk in ${CLOUDIATR_DISKS}; do + cloudiatr_fyi "geli-detaching ${disk}${partition_id}" + geli detach "${disk}${partition_id}.eli" + done +} + +cloudiatr_get_bpool_geoms() { + cloudiatr_get_geoms "p${CLOUDIATR_BPOOL_PARTITION}" +} + +cloudiatr_get_disk_names() { + local disk + for disk in ${CLOUDIATR_DISKS}; do + echo "${disk##*/}" + done +} + +cloudiatr_get_number_of_disks() { + local \ + disk number_of_disks + + number_of_disks=0 + for disk in ${CLOUDIATR_DISKS}; do + number_of_disks=$((number_of_disks+1)) + done + echo "${number_of_disks}" +} + +cloudiatr_get_default_pool_layout() { + if [ "$(cloudiatr_get_number_of_disks)" = 1 ]; then + echo "single-disk" + else + echo "mirror" + fi +} + +cloudiatr_autodetect_disks() { + local \ + disk + + for disk in $(sysctl -n kern.disks); do + # Only use ada(4) devices. We obviously can't use cd(4) + # devices and using da(4) devices would require us to + # skip the one we (probably) booted from. + if [ "${disk##ada}" != "${disk}" ]; then + echo "/dev/${disk}" + fi + done +} + +cloudiatr_create_bpool() { + local \ + bpool_geoms pool_layout + + bpool_geoms="$(cloudiatr_get_bpool_geoms)" + pool_layout="${CLOUDIATR_BPOOL_LAYOUT}" + + if [ "${pool_layout}" = "default" ]; then + pool_layout="$(cloudiatr_get_default_pool_layout)" + fi + + cloudiatr_fyi "Creating boot pool '${CLOUDIATR_BPOOL_NAME}' on ${bpool_geoms}. Pool layout: ${pool_layout}" + + zpool create -f -o version=28 -O compression=lzjb \ + "${CLOUDIATR_BPOOL_NAME}" ${pool_layout##single-disk} \ + $bpool_geoms + + # This currently can't be set at create-time + zpool set "bootfs=${CLOUDIATR_BPOOL_NAME}" "${CLOUDIATR_BPOOL_NAME}" + + # Would be nice, but for the bootfs to work, + # its ./boot directory can't be a zfs fs. + # + # XXX: can we work around this by setting bootfs + # on bpool/boot and use a symlink from bootf/boot/boot + # to bootf/boot? + # zfs create "${CLOUDIATR_BPOOL_NAME}/boot" +} + +cloudiatr_setup_tmpfs() { + mkdir -p "${CLOUDIATR_NEW_SYSTEM_DIR}" + mount -t tmpfs tmpfs "${CLOUDIATR_NEW_SYSTEM_DIR}" +} + +cloudiatr_generate_rc_conf() { + local \ + netmask + + cat < "${config_file}" +} + +cloudiatr_create_config_files() { + local chroot_dir="${1}" + + cloudiatr_generate_file loader_conf "${chroot_dir}/boot/loader.conf" + cloudiatr_generate_file sysctl_conf "${chroot_dir}/etc/sysctl.conf" + cloudiatr_generate_file rc_conf "${chroot_dir}/etc/rc.conf" + cloudiatr_generate_file fstab "${chroot_dir}/etc/fstab" + cloudiatr_generate_file resolv_conf "${chroot_dir}/etc/resolv.conf" || true +} + +cloudiatr_get_required_kernel_content() { + kldstat | awk '/k/ {print $5}' +} + +# XXX: May creates output with duplicated slashes. Ugly but harmless. +# XXX: Why do we ignore errors here? +cloudiatr_populate_bpool() { + local boot_dir \ + boot_file new_kernel_dir sub_dir bpool_mountpoint new_file + + boot_dir="${1}" + if [ "${boot_dir}" = "/" ]; then + # Prevent duplicated leading slash in log messages + boot_dir="" + fi + bpool_mountpoint="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_BPOOL_NAME}" + + cloudiatr_fyi "Populating boot pool '${CLOUDIATR_BPOOL_NAME}' ..." + zfs set mountpoint="${bpool_mountpoint}" "${CLOUDIATR_BPOOL_NAME}" || return 1 + + # Only copy what we really need before we can import the encrypted rpool + for sub_dir in dtb firmware kernel kernel.old zfs modules defaults; do + mkdir -p "${bpool_mountpoint}/boot/${sub_dir}" + done + + for boot_file in $(cloudiatr_get_required_kernel_content); do + # XXX: Create missing directories here + cp -v "${boot_dir}/boot/kernel/${boot_file}" "${bpool_mountpoint}/boot/kernel/" || true + if [ -f "${boot_dir}/boot/kernel.old/${boot_file}" ]; then + cp -v "${boot_dir}/boot/kernel.old/${boot_file}" "${bpool_mountpoint}/boot/kernel.old/" || true + fi + done + + for boot_file in $(find "${boot_dir}/boot/" \ + -not -path "${boot_dir}/boot/kernel*" -a \ + -not -path "${boot_dir}/boot/boot*" -a \ + -not -path "${boot_dir}/boot/cdboot" -a \ + -not -path "${boot_dir}/boot/*mbr" -a \ + -type f); do + new_file="${bpool_mountpoint}/boot/${boot_file##*/boot/}" + cp -v "${boot_file}" "${new_file}" || true + done + + zfs set "mountpoint=/${CLOUDIATR_BPOOL_NAME}" "${CLOUDIATR_BPOOL_NAME}" +} + +cloudiatr_clean_up() { + cloudiatr_fyi "Exporting boot pool '${CLOUDIATR_BPOOL_NAME}' ..." + zpool export "${CLOUDIATR_BPOOL_NAME}" + cloudiatr_fyi "Exporting root pool '${CLOUDIATR_RPOOL_NAME}' ..." + zpool export "${CLOUDIATR_RPOOL_NAME}" + cloudiatr_detach_geli_geoms "p${CLOUDIATR_RPOOL_PARTITION}" + umount "${CLOUDIATR_NEW_SYSTEM_DIR}" + rmdir "${CLOUDIATR_NEW_SYSTEM_DIR}" + if "${CLOUDIATR_USE_SWAP_WHILE_INSTALLING}"; then + if ! cloudiatr_disable_swap ${CLOUDIATR_DISKS}; then + cloudiatr_fyi "Failed to remove all the added swap space." + cloudiatr_fyi "If the system is low on memory the problem can be safely ignored." + fi + fi +} + +cloudiatr_generate_ssh_hostkeys() { + local chroot_dir \ + real_hostname + + chroot_dir="${1}" + real_hostname="$(hostname)" + + hostname "${CLOUDIATR_HOSTNAME}" + cloudiatr_fyi "Generating ssh host keys for ${CLOUDIATR_HOSTNAME} ..." + for key_alg in ${CLOUDIATR_SSHD_HOST_KEY_ALGORITHMS}; do + key_file="${chroot_dir}/etc/ssh/ssh_host_${key_alg}_key" + if ! ssh-keygen -q -t "${key_alg}" -f "${key_file}" -N ""; then + if [ "${key_alg}" = "ed25519" ]; then + # ed25519 isn't supported on FreeBSD 10.0 and earlier, + # thus we allow this to fail + continue + fi + return 1 + fi + ssh-keygen -l -v -f "${key_file}.pub" + done + hostname "${real_hostname}" +} + +cloudiatr_collect_evidence() { + local \ + evidence_dataset evidence disk_name + + evidence_dataset="${CLOUDIATR_RPOOL_NAME}/cloudiatr-evidence" + + cloudiatr_fyi "Collecting 'evidence' in /${evidence_dataset} ..." + zfs create "${evidence_dataset}" + for evidence in "${CLOUDIATR_CONFIG_FILE}" "${0}" \ + "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups/"* \ + "${CLOUDIATR_RPOOL_KEY}"; do + cp -p "${evidence}" "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}" + done + + for disk_name in $(cloudiatr_get_disk_names); do + evidence="${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}/${disk_name}.gpart" + gpart backup "${disk_name}" > "${evidence}" + done + + if "${CLOUDIATR_SAVE_DIST_DIR}"; then + cp -r "${CLOUDIATR_DIST_DIR%%/}" "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}/" + fi + + find "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}" -type f | sort +} + +cloudiatr_generate_sshd_config_extension() { + cat <> ${chroot_dir}/etc/ssh/sshd_config + + if "${CLOUDIATR_USE_NTPDATE}"; then + chroot ${chroot_dir} service ntpdate onestart || true + fi + + umount ${chroot_dir}/dev/ + + cloudiatr_setup_new_user + + # Copying the zpool.cache is no longer necessary on ElectroBSD + # and recent FreeBSD versions, but doesn't hurt. + cp /boot/zfs/zpool.cache "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/boot/zfs/zpool.cache" + + cloudiatr_populate_bpool "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}" + + cloudiatr_collect_evidence + + cloudiatr_fyi "Setting final mountpoints on root pool '${CLOUDIATR_RPOOL_NAME}' ..." + zfs umount "${CLOUDIATR_RPOOL_NAME}" + zfs set mountpoint=legacy "${CLOUDIATR_RPOOL_NAME}" + for fs in boot home tmp usr var; do + zfs set "mountpoint=/${fs}" "${CLOUDIATR_RPOOL_NAME}/${fs}" + done + + cloudiatr_clean_up +} + +# The murder-death-kill feature was added for testing. After the introduction of +# the boring "clean-up" subcommand (which doesn't involve killing) it could be +# considered obsolete, but keeping it around makes cloudiatr more awesome. +cloudiatr_murder_death_kill() { + if "${CLOUDIATR_MURDER_DEATH_KILL_REQUESTED}"; then + cloudiatr_fyi "You really asked for it. Murder death kill in progress ..." + zpool export "${CLOUDIATR_BPOOL_NAME}" || true + zpool export "${CLOUDIATR_RPOOL_NAME}" || true + umount "${CLOUDIATR_NEW_SYSTEM_DIR}" || true + geli kill -a || true + fi +} + +cloudiatr_request_consent() { + local message="${*}" \ + response + + if "${CLOUDIATR_DONT_ASK_JUST_KISS}"; then + # ... the data goodbye. + return 0 + fi + + echo -n "cloudiatr: ${message} [y/n] " + # XXX: Don't use "read -p" as it may work unreliably + read response + + # "No" means "no". Everything but "y" also means "no". + [ "${response}" = "y" ] +} + +cloudiatr_has_eviction_consent() { + echo "cloudiatr (${CLOUDIATR_VERSION}) can't wait to evict '$(hostname)' ..." + echo + echo "Depending on your jurisdiction, 'eviction without consent' may be against the law." + echo "cloudiatr doesn't bother to make backups of the existing data. That's what zogftw is for." + echo + cloudiatr_request_consent "Continue eviction?" +} + +cloudiatr_evict() { + if cloudiatr_has_eviction_consent; then + if "${CLOUDIATR_MURDER_DEATH_KILL_REQUESTED}"; then + cloudiatr_murder_death_kill + else + cloudiatr_fyi "You asked for it ..." + fi + cloudiatr_fyi "Eviction in progress ..." + cloudiatr_evict_local_system + cloudiatr_fyi "Looks like somebody managed to install a real operating system ..." + if cloudiatr_request_consent "Reboot now?"; then + shutdown -r now + fi + return 0 + else + cloudiatr_fyi "Eviction aborted in time ..." + return 1 + fi +} + +cloudiatr_has_soft_protect_consent() { + cloudiatr_request_consent "Put $(uname) in 'Soviet Germany' mode?" +} + +# XXX: Only works for the cloudiatr disk layout. +cloudiatr_soft_protect() { + local \ + mirror_name device_to_clear number_of_disks + + mirror_name="vdev-remains" + number_of_disks="$(cloudiatr_get_number_of_disks)" + + cloudiatr_fyi "Destroying ${CLOUDIATR_BPOOL_NAME} ..." + cloudiatr_fyi "Use 'geli kill -a' to 'hard-protect' your data right now. No recovery without remote backups!" + + if zpool list "${CLOUDIATR_BPOOL_NAME}" >/dev/null 2>&1; then + zpool destroy "${CLOUDIATR_BPOOL_NAME}" || true + fi + + if [ "${number_of_disks}" = 1 ]; then + cloudiatr_fyi "Nuking former ${CLOUDIATR_BPOOL_NAME} vdev from orbit ..." + device_to_clear="$(cloudiatr_get_bpool_geoms)" + else + cloudiatr_fyi "Nuking former ${CLOUDIATR_BPOOL_NAME} vdevs from orbit using gmirror power ..." + gmirror load 2>/dev/null || true + gmirror label "${mirror_name}" $(cloudiatr_get_bpool_geoms) + device_to_clear="/dev/mirror/${mirror_name}" + fi + + geli onetime -s 4096 -e "${CLOUDIATR_GELI_EALGO}" "${device_to_clear}" + dd if=/dev/zero bs=1M of="${device_to_clear}.eli" 2>/dev/null || true + geli detach "${device_to_clear##/dev/}" + + cloudiatr_fyi "Done. Levelling nuked wasteland with zeroes ..." + dd if=/dev/zero bs=1M of="${device_to_clear}" 2>/dev/null || true + + if [ "${number_of_disks}" != 1 ]; then + gmirror destroy "${mirror_name}" + fi + cloudiatr_fyi "Done. $(uname) should remain working as expected until the next shutdown ..." + cloudiatr_fyi "Remember to 'unprotect' the system before consensual reboots (or use the opportunity to test your backup system)" +} + +cloudiatr_check_privileges() { + local \ + uid user + + uid="$(id -u)" + user="$(id -un)" + + if [ "${uid}" != 0 ]; then + cloudiatr_wtf "Check your privileges, $user. It looks like you might not have enough of them!" + cloudiatr_fyi "Hint: Using 'sudo' or 'su' might help." + return 1 + fi + + return 0 +} + +cloudiatr_usage() { + local \ + subcommand + + for subcommand in clean-up cmd recreate-bpool evict soft-protect show-config; do + echo "cloudiatr [-f config-file] $subcommand" + done +} + +cloudiatr_update_base() { + local \ + base_dist base_dbg_dist + + base_dist="${CLOUDIATR_DIST_DIR}/base.txz" + base_dbg_dist="${CLOUDIATR_DIST_DIR}/base-dbg.txz" + + if [ ! -f "${base_dist}" ]; then + cloudiatr_wtf "Base update impossible. '${base_dist}' does not exist" + return 1 + fi + cloudiatr_wtf "Updating base system ..." + chflags -R noschg /bin/ /lib/ /sbin/ /usr/bin/ /usr/lib /libexec/ || return 1 + cd / || return 1 + tar xpf "${base_dist}" \ + --exclude ./etc/ --exclude ./var/empty --exclude ./usr/src || return 1 + if [ -f "${base_dbg_dist}" ]; then + tar xpf "${base_dbg_dist}" || return 1 + fi +} + +cloudiatr_main() { + local mode + + if [ "${1}" = "-f" ]; then + shift + CLOUDIATR_CONFIG_FILE="${1}" + shift + if [ -z "${CLOUDIATR_CONFIG_FILE}" ] || + ! [ -f "${CLOUDIATR_CONFIG_FILE}" ]; then + cloudiatr_wtf "No existing config file specified." + return 1 + fi + fi + + mode="${1}" + + if [ "${mode}" != "cmd" ] && [ "${mode}" != "show-config" ]; then + cloudiatr_check_privileges || return 1 + fi + + cloudiatr_init "${mode}" + + case "${mode}" in + clean-up) + set +e + cloudiatr_clean_up + ;; + cmd) + shift + "${@}" + ;; + recreate-bpool) + cloudiatr_create_bpool + cloudiatr_populate_bpool "/" + ;; + evict) + cloudiatr_evict + ;; + soft-protect) + if cloudiatr_has_soft_protect_consent; then + cloudiatr_soft_protect + fi + ;; + update) + cloudiatr_update_base || return 1 + ;; + show-config) + cloudiatr_show_config + ;; + #rekey) + # XXX: Not yet implemented + *) + cloudiatr_usage + cloudiatr_wtf "Invalid or missing subcommand" + return 1 + ;; + esac + +} + +cloudiatr_main "${@}" -- 2.11.0 From 0506983fc2bea4a6df4dc7431755b2d0763fd0dd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Dec 2016 12:25:38 +0100 Subject: [PATCH 251/255] Detach telnetd from the build Obtained from: ElectroBSD --- secure/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secure/Makefile b/secure/Makefile index c3b2c87e1462..98784c830ad8 100644 --- a/secure/Makefile +++ b/secure/Makefile @@ -12,7 +12,7 @@ _tests= tests # These are the programs which depend on crypto, but not Kerberos. SPROGS= lib/libfetch lib/libpam lib/libradius lib/libtelnet \ - bin/ed libexec/telnetd usr.bin/fetch usr.bin/telnet \ + bin/ed usr.bin/fetch usr.bin/telnet \ usr.sbin/pkg_install usr.sbin/ppp usr.sbin/tcpdump/tcpdump .if ${MK_SENDMAIL} != "no" SPROGS+=usr.sbin/sendmail -- 2.11.0 From d8a79f804cc3fa0244fd3f0d95605d77e6ddd4e2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 18 Dec 2016 16:03:52 +0100 Subject: [PATCH 252/255] release: Let the src.txz target enforce the file and directory mode Obtained from: ElectroBSD --- release/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/Makefile b/release/Makefile index 27d47434c416..9d2878bddcec 100644 --- a/release/Makefile +++ b/release/Makefile @@ -181,6 +181,8 @@ src.txz: cd ${DISTDIR} && mtree -c -L -k time -X ${.OBJDIR}/mtree-exclude | \ mtree -C | \ sed -E -e 's@time=[0-9]+\.[0-9]+@time=${EPOCH_DATE}.0 uid=0 gid=0@' \ + -e 's@(type=dir)@\1 mode=0550@' \ + -e 's@(type=file)@\1 mode=0440@' \ > ${.OBJDIR}/src.mtree rm ${.OBJDIR}/mtree-exclude cd ${DISTDIR} && tar cLvf - @${.OBJDIR}/src.mtree \ -- 2.11.0 From 2cd47b084759e32f9518aecb8cfa12b77ee17ed3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 11 Dec 2016 16:15:58 +0100 Subject: [PATCH 253/255] Import reproduce-electrobsd.sh 2016-12-11-24c1a6b Obtained from: ElectroBSD --- reproduce-electrobsd.sh | 141 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 reproduce-electrobsd.sh diff --git a/reproduce-electrobsd.sh b/reproduce-electrobsd.sh new file mode 100644 index 000000000000..94cad4cfdb16 --- /dev/null +++ b/reproduce-electrobsd.sh @@ -0,0 +1,141 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2016 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# reproduce-electrobsd.sh +# +# This script builds ElectroBSD in a jail, updates the jail +# using the built binaries and builds ElectroBSD again. +# +# The second build should thus always result with the same +# binaries, even if different ElectroBSD versions were used +# for the first build, or (not yet tested) if the first build +# was done with FreeBSD binaries. +# +# The following steps are not yet automated: +# +# 1) Create an /etc/jail.conf with a section like this one: +# +# ElectroBSD-amd64 { +# host.hostname=ElectroBSD-amd64; +# persist; +# children.max=0; +# allow.mount; +# allow.mount.devfs; +# enforce_statfs=1; +# path=/usr/jails/ElectroBSD-amd64; +# exec.start="mount -t devfs devfs /dev"; +# exec.stop="umount /dev"; +# } +# +# 2) Create two new ZFS datasets for the build jail: +# +# /usr/jails/ElectroBSD-amd64 +# /usr/jails/ElectroBSD-amd64/usr +# +# 3) Extract a base.txz in it. +# +# 4) Mount an UFS file system in "/usr/jails/ElectroBSD-amd64/usr/obj" +# and allow the BUILD_USER to run it. This step is only required +# to get reproducible images, the tarballs should be reproducible +# when using ZFS as well. +# +# 5) Create a dedicated /usr/src dataset on the host and populate it +# with src.txz +# +# 6) Install sudo and zogftw on the host. + +HOST_SRC_DIR="/usr/src" +HOST_BUILDLOG_DIR="/var/log/buildlogs" + +BUILD_JAIL="${BUILD_JAIL-ElectroBSD-amd64}" +BUILD_JAIL_ROOT="/usr/jails/${BUILD_JAIL}" +BUILD_JAIL_SRC_DIR="${BUILD_JAIL_ROOT}/usr/src" +BUILD_USER="$(id -un)" +BUILD_NICE_VALUE=20 +BUILD_CORE_COUNT=2 +# This is relative to the BUILD_JAIL_ROOT +BUILD_DISTFILE_DIR_PREFIX=/usr/obj/usr/src/ + +prepare_build_jail() { + local \ + src_dataset last_src_snapshot src_clone build_jail_dataset + + # Use vanilla zogftw configuration without potentionally + # existing fancy custom hooks that could slow use down. + export ZOGFTW_CONFIG_FILE='' + + zogftw snap /usr/src + + src_dataset=$(zogftw zcmd get_dataset_from_path "${HOST_SRC_DIR}") + if [ -z "${src_dataset}" ]; then + echo "Failed to get src dataset" + return 1 + fi + src_clone=$(zogftw zcmd get_dataset_from_path "${BUILD_JAIL_SRC_DIR}") + if [ -n "${src_clone}" ]; then + # The build jail already has a clone /usr/src, + # delete it as it may be stale + sudo zfs destroy "${src_clone}" || return 1 + else + build_jail_dataset=$(zogftw zcmd get_dataset_from_path "${BUILD_JAIL_ROOT}") + if [ -z "${build_jail_dataset}" ]; then + echo "Failed to get root jail dataset" + return 1 + fi + src_clone="${build_jail_dataset}/usr/src" + echo "Will create fresh clone in $src_clone" + fi + last_src_snapshot=$(zogftw zcmd get_last_snapshot "${src_dataset}") + if [ -z "${last_src_snapshot}" ]; then + echo "Failed to get src clone" + return 1 + fi + sudo zfs clone "${last_src_snapshot}" "${src_clone}" || return 1 +} + +reproduce() { + local \ + timestamp logfile build distfile_dir_prefix + + timestamp=$(date "+%Y-%m-%d_%H:%M") + logfile="${HOST_BUILDLOG_DIR}/reprolog-${timestamp}" + build="$(grep ^BUILD= /usr/src/reproduce.conf | cut -d = -f 2)" + distfile_dir_prefix="/usr/obj/usr/src/$build" + + script "${logfile}" \ + sudo nice -n "${BUILD_NICE_VALUE}" \ + sh -c "jail -c '${BUILD_JAIL}' && \ + jexec -u ${BUILD_USER} '${BUILD_JAIL}' \ + /usr/src/reproduce.sh -j${BUILD_CORE_COUNT} -d ${distfile_dir_prefix}-j1 -a && \ + jexec '${BUILD_JAIL}' make -C /usr/src installworld NO_FSCHG='yes' && \ + jexec -u ${BUILD_USER} '${BUILD_JAIL}' \ + /usr/src/reproduce.sh -j${BUILD_CORE_COUNT} -d ${distfile_dir_prefix}-j2; + jail -r '${BUILD_JAIL}'" || return 1 + + # Compare checksums. Use a separate script call to make + # sure the checksums from the second run are flushed to + # the file + script -a "${logfile}" sh -c "grep ^SHA256 '${logfile}' | sort -k 4 | column -t" +} + +main() { + prepare_build_jail || return 1 + reproduce || return 1 +} + +main -- 2.11.0 From 531a8a2a5831e06203458a86adc43ca3db16eb5b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 30 Dec 2016 16:54:28 +0100 Subject: [PATCH 254/255] g_dev_orphan(): Return early if the device is already gone Supposed to fix a panic that could occur while running "cdcontrol eject" after using the physical ejection key on the device: Unread portion of the kernel message buffer: stack pointer = 0x28:0xfffffe01eba0a9e0 frame pointer = 0x28:0xfffffe01eba0a9f0 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 13 (g_event) trap number = 12 panic: page fault cpuid = 3 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe01eba0a560 vpanic() at vpanic+0x182/frame 0xfffffe01eba0a5e0 panic() at panic+0x43/frame 0xfffffe01eba0a640 trap_fatal() at trap_fatal+0x331/frame 0xfffffe01eba0a6a0 trap_pfault() at trap_pfault+0x1e3/frame 0xfffffe01eba0a700 trap() at trap+0x273/frame 0xfffffe01eba0a910 calltrap() at calltrap+0x8/frame 0xfffffe01eba0a910 --- trap 0xc, rip = 0xffffffff80500fde, rsp = 0xfffffe01eba0a9e0, rbp = 0xfffffe01eba0a9f0 --- g_dev_orphan() at g_dev_orphan+0x2e/frame 0xfffffe01eba0a9f0 g_resize_provider_event() at g_resize_provider_event+0x71/frame 0xfffffe01eba0aa20 g_run_events() at g_run_events+0x20e/frame 0xfffffe01eba0aa70 fork_exit() at fork_exit+0x85/frame 0xfffffe01eba0aab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01eba0aab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Uptime: 3h17m41s Dumping 1120 out of 8055 MB:..2%..12%..22%..32%..42%..52%..62%..72%..82%..92% Reading symbols from /usr/lib/debug/boot/kernel/zfs.ko.debug...done. [...] Loaded symbols for /usr/lib/debug/boot/kernel/iicbb.ko.debug #0 doadump (textdump=1) at pcpu.h:222 222 pcpu.h: No such file or directory. in pcpu.h (kgdb) where #0 doadump (textdump=1) at pcpu.h:222 #1 0xffffffff805cce3e in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:366 #2 0xffffffff805cd40b in vpanic (fmt=, ap=) at /usr/src/sys/kern/kern_shutdown.c:759 #3 0xffffffff805cd243 in panic (fmt=0x0) at /usr/src/sys/kern/kern_shutdown.c:690 #4 0xffffffff808eece1 in trap_fatal (frame=0xfffffe01eba0a920, eva=8) at /usr/src/sys/amd64/amd64/trap.c:801 #5 0xffffffff808eeed3 in trap_pfault (frame=0xfffffe01eba0a920, usermode=0) at /usr/src/sys/amd64/amd64/trap.c:658 #6 0xffffffff808ee4d3 in trap (frame=0xfffffe01eba0a920) at /usr/src/sys/amd64/amd64/trap.c:421 #7 0xffffffff808d2701 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:236 #8 0xffffffff80500fde in g_dev_orphan (cp=0xfffff800069e6280) at /usr/src/sys/geom/geom_dev.c:754 #9 0xffffffff80509ff1 in g_resize_provider_event (arg=, flag=) at /usr/src/sys/geom/geom_subr.c:631 #10 0xffffffff80504f1e in g_run_events () at /usr/src/sys/geom/geom_event.c:264 #11 0xffffffff805830e5 in fork_exit (callout=0xffffffff805079c0 , arg=0x0, frame=0xfffffe01eba0aac0) at /usr/src/sys/kern/kern_fork.c:1040 #12 0xffffffff808d2c3e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:611 #13 0x0000000000000000 in ?? () Current language: auto; currently minimal (kgdb) f 8 #8 0xffffffff80500fde in g_dev_orphan (cp=0xfffff800069e6280) at /usr/src/sys/geom/geom_dev.c:754 754 g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); (kgdb) p cp $1 = (struct g_consumer *) 0xfffff800069e6280 (kgdb) p cp->geom $2 = (struct g_geom *) 0xfffff801acd6d100 (kgdb) p cp->geom->name $3 = 0xfffff8007479bf60 "cd0" (kgdb) f 8 #8 0xffffffff80500fde in g_dev_orphan (cp=0xfffff800069e6280) at /usr/src/sys/geom/geom_dev.c:754 754 g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); (kgdb) l 754 g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); 755 756 /* Reset any dump-area set on this device */ 757 if (dev->si_flags & SI_DUMPDEV) 758 (void)set_dumper(NULL, NULL, curthread); 759 760 /* Destroy the struct cdev *so we get no more requests */ 761 destroy_dev_sched_cb(dev, g_dev_callback, cp); 762 } 763 (kgdb) p dev->si_flags Cannot access memory at address 0x8 (kgdb) p dev $4 = (struct cdev *) 0x0 Last message before the panic: Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:33:49 t520 kernel: ahcich1: Timeout on slot 8 port 0 Dec 30 16:33:49 t520 kernel: ahcich1: is 00000000 cs 00000100 ss 00000000 rs 00000100 tfd d0 serr 00000000 cmd 000cc817 Dec 30 16:33:49 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:33:49 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: Command timeout Dec 30 16:33:49 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: MEDIUM ERROR asc:10,0 (ID CRC or ECC error) Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ad 00 00 04 00 Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: MEDIUM ERROR asc:10,0 (ID CRC or ECC error) Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ad 00 00 04 00 Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: MEDIUM ERROR asc:10,0 (ID CRC or ECC error) Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ea 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ea 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=215856 PR submission date: 2017-01-07 Obtained from: ElectroBSD --- sys/geom/geom_dev.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sys/geom/geom_dev.c b/sys/geom/geom_dev.c index d74b92e59776..543df1db3614 100644 --- a/sys/geom/geom_dev.c +++ b/sys/geom/geom_dev.c @@ -753,6 +753,9 @@ g_dev_orphan(struct g_consumer *cp) dev = sc->sc_dev; g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); + if (dev == NULL) + return; + /* Reset any dump-area set on this device */ if (dev->si_flags & SI_DUMPDEV) (void)set_dumper(NULL, NULL, curthread); -- 2.11.0 From d3445bb4d843899aba4e469c748f1d98afc074ea Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 8 Jan 2017 12:48:25 +0100 Subject: [PATCH 255/255] Add reproduce.conf for ElectroBSD-r311643-531a8a2a5831 Expected checksums when building with itself on amd64: SHA256 (MANIFEST) = 605822f30411eba27e0cdf041b430992df6c2f065323152300904c461b506e98 SHA256 (base-dbg.txz) = 958e018ca8cd976b4d07fd18d0db0538342237890cb65b42ac6865a6f4cf5c29 SHA256 (base.txz) = dcd6b1a9c003b04fc8526627dbdc1d5e3af154f423e35e8f7147611b004c55bf SHA256 (kernel-dbg.txz) = 34d3179f5b44d69f230327cc3e7c394e713a231fd3c9e9e764e2841c07a5e6b3 SHA256 (kernel.txz) = 8cef9f4ba529e3caa5cdef98be01993fffc2274de21c59cc8588940bf44470a6 SHA256 (src.txz) = be3917c7432d70e0c03c1b8824c96d278e0aa7ff56a2ab63aee8bb910431839e SHA256 (tests.txz) = 57ae1bb376c7fe87555df8e6d6db8fda218a7451d5f2595af8476b0f7447f481 Reproducibily of the memstick image depends on external factors that haven't been analyzed yet. On two systems with 4 GB RAM the checksum was: SHA256 (ElectroBSD-r311643-531a8a2a5831.img) = c4df2fe46c582b19f36793eb59dce078d0b571cf2c7fe15322df8a17b975ff66 Obtained from: ElectroBSD --- reproduce.conf | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 reproduce.conf diff --git a/reproduce.conf b/reproduce.conf new file mode 100644 index 000000000000..c6587fa07acf --- /dev/null +++ b/reproduce.conf @@ -0,0 +1,2 @@ +BUILD=ElectroBSD-r311643-531a8a2a5831 +EPOCH=1483789724 -- 2.11.0