From a0c598036fcb883eff873896b852c1ec93d0dc22 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:24:47 +0200 Subject: [PATCH 001/213] amd64: Save a copy of GENERIC as ELECTRO_BLOAT --- sys/amd64/conf/ELECTRO_BLOAT | 365 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 sys/amd64/conf/ELECTRO_BLOAT diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT new file mode 100644 index 0000000..3d2cc39 --- /dev/null +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -0,0 +1,365 @@ +# +# GENERIC -- Generic kernel configuration file for FreeBSD/amd64 +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +cpu HAMMER +ident GENERIC + +makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols +makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support + +options SCHED_ULE # ULE scheduler +options PREEMPTION # Enable kernel thread preemption +options INET # InterNETworking +options INET6 # IPv6 communications protocols +options TCP_OFFLOAD # TCP offload +options SCTP # Stream Control Transmission Protocol +options FFS # Berkeley Fast Filesystem +options SOFTUPDATES # Enable FFS soft updates support +options UFS_ACL # Support for access control lists +options UFS_DIRHASH # Improve performance on big directories +options UFS_GJOURNAL # Enable gjournal-based UFS journaling +options QUOTA # Enable disk quotas for UFS +options MD_ROOT # MD is a potential root device +options NFSCL # Network Filesystem Client +options NFSD # Network Filesystem Server +options NFSLOCKD # Network Lock Manager +options NFS_ROOT # NFS usable as /, requires NFSCL +options MSDOSFS # MSDOS Filesystem +options CD9660 # ISO 9660 Filesystem +options PROCFS # Process filesystem (requires PSEUDOFS) +options PSEUDOFS # Pseudo-filesystem framework +options GEOM_PART_GPT # GUID Partition Tables. +options GEOM_RAID # Soft RAID functionality. +options GEOM_LABEL # Provides labelization +options COMPAT_FREEBSD32 # Compatible with i386 binaries +options COMPAT_FREEBSD4 # Compatible with FreeBSD4 +options COMPAT_FREEBSD5 # Compatible with FreeBSD5 +options COMPAT_FREEBSD6 # Compatible with FreeBSD6 +options COMPAT_FREEBSD7 # Compatible with FreeBSD7 +options COMPAT_FREEBSD9 # Compatible with FreeBSD9 +options COMPAT_FREEBSD10 # Compatible with FreeBSD10 +options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI +options KTRACE # ktrace(1) support +options STACK # stack(9) support +options SYSVSHM # SYSV-style shared memory +options SYSVMSG # SYSV-style message queues +options SYSVSEM # SYSV-style semaphores +options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions +options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. +options KBD_INSTALL_CDEV # install a CDEV entry in /dev +options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) +options AUDIT # Security event auditing +options CAPABILITY_MODE # Capsicum capability mode +options CAPABILITIES # Capsicum capabilities +options MAC # TrustedBSD MAC Framework +options KDTRACE_FRAME # Ensure frames are compiled in +options KDTRACE_HOOKS # Kernel DTrace hooks +options DDB_CTF # Kernel ELF linker loads CTF data +options INCLUDE_CONFIG_FILE # Include this file in kernel +options RACCT # Resource accounting framework +options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default +options RCTL # Resource limits + +# Debugging support. Always need this: +options KDB # Enable kernel debugger support. +options KDB_TRACE # Print a stack trace for a panic. +# For full debugger support use (turn off in stable branch): +options DDB # Support DDB. +options GDB # Support remote GDB. +options DEADLKRES # Enable the deadlock resolver +options INVARIANTS # Enable calls of extra sanity checking +options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +options WITNESS # Enable checks to detect deadlocks and cycles +options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones + +# Make an SMP-capable kernel by default +options SMP # Symmetric MultiProcessor Kernel + +# CPU frequency control +device cpufreq + +# Bus support. +device acpi +options ACPI_DMAR +device pci +options PCI_IOV # PCI SR-IOV support + +# Floppy drives +device fdc + +# ATA controllers +device ahci # AHCI-compatible SATA controllers +device ata # Legacy ATA/SATA controllers +options ATA_STATIC_ID # Static device numbering +device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +device ahc # AHA2940 and onboard AIC7xxx devices +options AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +device ahd # AHA39320/29320 and onboard AIC79xx devices +options AHD_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~215k to driver. +device esp # AMD Am53C974 (Tekram DC-390(T)) +device hptiop # Highpoint RocketRaid 3xxx series +device isp # Qlogic family +#device ispfw # Firmware for QLogic HBAs- normally a module +device mpt # LSI-Logic MPT-Fusion +device mps # LSI-Logic MPT-Fusion 2 +device mpr # LSI-Logic MPT-Fusion 3 +#device ncr # NCR/Symbios Logic +device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +device trm # Tekram DC395U/UW/F DC315U adapters + +device adv # Advansys SCSI adapters +device adw # Advansys wide SCSI adapters +device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +device bt # Buslogic/Mylex MultiMaster SCSI adapters +device isci # Intel C600 SAS controller + +# ATA/SCSI peripherals +device scbus # SCSI bus (required for ATA/SCSI) +device ch # SCSI media changers +device da # Direct Access (disks) +device sa # Sequential Access (tape etc) +device cd # CD +device pass # Passthrough device (direct ATA/SCSI access) +device ses # Enclosure Services (SES and SAF-TE) +#device ctl # CAM Target Layer + +# RAID controllers interfaced to the SCSI subsystem +device amr # AMI MegaRAID +device arcmsr # Areca SATA II RAID +device ciss # Compaq Smart RAID 5* +device dpt # DPT Smartcache III, IV - See NOTES for options +device hptmv # Highpoint RocketRAID 182x +device hptnr # Highpoint DC7280, R750 +device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +device hpt27xx # Highpoint RocketRAID 27xx +device iir # Intel Integrated RAID +device ips # IBM (Adaptec) ServeRAID +device mly # Mylex AcceleRAID/eXtremeRAID +device twa # 3ware 9000 series PATA/SATA RAID +device tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +device aac # Adaptec FSA RAID +device aacp # SCSI passthrough for aac (requires CAM) +device aacraid # Adaptec by PMC RAID +device ida # Compaq Smart RAID +device mfi # LSI MegaRAID SAS +device mlx # Mylex DAC960 family +device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s +#XXX pointer/int warnings +#device pst # Promise Supertrak SX6000 +device twe # 3ware ATA RAID + +# NVM Express (NVMe) support +device nvme # base NVMe driver +device nvd # expose NVMe namespaces as disks, depends on nvme + +# atkbdc0 controls both the keyboard and the PS/2 mouse +device atkbdc # AT keyboard controller +device atkbd # AT keyboard +device psm # PS/2 mouse + +device kbdmux # keyboard multiplexer + +device vga # VGA video card driver +options VESA # Add support for VESA BIOS Extensions (VBE) + +device splash # Splash screen and screen saver support + +# syscons is the default console driver, resembling an SCO console +device sc +options SC_PIXEL_MODE # add support for the raster text mode + +# vt is the new video console driver +device vt +device vt_vga +device vt_efifb + +device agp # support several AGP chipsets + +# PCCARD (PCMCIA) support +# PCMCIA and cardbus bridge support +device cbb # cardbus (yenta) bridge +device pccard # PC Card (16-bit) bus +device cardbus # CardBus (32-bit) bus + +# Serial (COM) ports +device uart # Generic UART driver + +# Parallel port +device ppc +device ppbus # Parallel port bus (required) +device lpt # Printer +device ppi # Parallel port interface device +#device vpo # Requires scbus and da + +device puc # Multi I/O cards and multi-channel UARTs + +# PCI Ethernet NICs. +device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +device de # DEC/Intel DC21x4x (``Tulip'') +device em # Intel PRO/1000 Gigabit Ethernet Family +device igb # Intel PRO/1000 PCIE Server Gigabit Family +device ix # Intel PRO/10GbE PCIE PF Ethernet +device ixv # Intel PRO/10GbE PCIE VF Ethernet +device ixl # Intel XL710 40Gbe PCIE Ethernet +device ixlv # Intel XL710 40Gbe VF PCIE Ethernet +device le # AMD Am7900 LANCE and Am79C9xx PCnet +device ti # Alteon Networks Tigon I/II gigabit Ethernet +device txp # 3Com 3cR990 (``Typhoon'') +device vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! +device miibus # MII bus support +device ae # Attansic/Atheros L2 FastEthernet +device age # Attansic/Atheros L1 Gigabit Ethernet +device alc # Atheros AR8131/AR8132 Ethernet +device ale # Atheros AR8121/AR8113/AR8114 Ethernet +device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +device bfe # Broadcom BCM440x 10/100 Ethernet +device bge # Broadcom BCM570xx Gigabit Ethernet +device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +device dc # DEC/Intel 21143 and various workalikes +device et # Agere ET1310 10/100/Gigabit Ethernet +device fxp # Intel EtherExpress PRO/100B (82557, 82558) +device gem # Sun GEM/Sun ERI/Apple GMAC +device hme # Sun HME (Happy Meal Ethernet) +device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +device lge # Level 1 LXT1001 gigabit Ethernet +device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +device nfe # nVidia nForce MCP on-board Ethernet +device nge # NatSemi DP83820 gigabit Ethernet +device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +device re # RealTek 8139C+/8169/8169S/8110S +device rl # RealTek 8129/8139 +device sf # Adaptec AIC-6915 (``Starfire'') +device sge # Silicon Integrated Systems SiS190/191 +device sis # Silicon Integrated Systems SiS 900/SiS 7016 +device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +device ste # Sundance ST201 (D-Link DFE-550TX) +device stge # Sundance/Tamarack TC9021 gigabit Ethernet +device tl # Texas Instruments ThunderLAN +device tx # SMC EtherPower II (83c170 ``EPIC'') +device vge # VIA VT612x gigabit Ethernet +device vr # VIA Rhine, Rhine II +device wb # Winbond W89C840F +device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# Wireless NIC cards +device wlan # 802.11 support +options IEEE80211_DEBUG # enable debug msgs +options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's +options IEEE80211_SUPPORT_MESH # enable 802.11s draft support +device wlan_wep # 802.11 WEP support +device wlan_ccmp # 802.11 CCMP support +device wlan_tkip # 802.11 TKIP support +device wlan_amrr # AMRR transmit rate control algorithm +device an # Aironet 4500/4800 802.11 wireless NICs. +device ath # Atheros NICs +device ath_pci # Atheros pci/cardbus glue +device ath_hal # pci/cardbus chip support +options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors +options AH_AR5416_INTERRUPT_MITIGATION # AR5416 interrupt mitigation +options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later +device ath_rate_sample # SampleRate tx rate control for ath +#device bwi # Broadcom BCM430x/BCM431x wireless NICs. +#device bwn # Broadcom BCM43xx wireless NICs. +device ipw # Intel 2100 wireless NICs. +device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +device iwn # Intel 4965/1000/5000/6000 wireless NICs. +device malo # Marvell Libertas wireless NICs. +device mwl # Marvell 88W8363 802.11n wireless NICs. +device ral # Ralink Technology RT2500 wireless NICs. +device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +device wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +device loop # Network loopback +device random # Entropy device +device padlock_rng # VIA Padlock RNG +device rdrand_rng # Intel Bull Mountain RNG +device ether # Ethernet support +device vlan # 802.1Q VLAN support +device tun # Packet tunnel. +device md # Memory "disks" +device gif # IPv6 and IPv4 tunneling +device firmware # firmware assist module + +# The `bpf' device enables the Berkeley Packet Filter. +# Be aware of the administrative consequences of enabling this! +# Note that 'bpf' is required for DHCP. +device bpf # Berkeley packet filter + +# USB support +options USB_DEBUG # enable debug msgs +device uhci # UHCI PCI->USB interface +device ohci # OHCI PCI->USB interface +device ehci # EHCI PCI->USB interface (USB 2.0) +device xhci # XHCI PCI->USB interface (USB 3.0) +device usb # USB Bus (required) +device ukbd # Keyboard +device umass # Disks/Mass storage - Requires scbus and da + +# Sound support +device sound # Generic sound driver (required) +device snd_cmi # CMedia CMI8338/CMI8738 +device snd_csa # Crystal Semiconductor CS461x/428x +device snd_emu10kx # Creative SoundBlaster Live! and Audigy +device snd_es137x # Ensoniq AudioPCI ES137x +device snd_hda # Intel High Definition Audio +device snd_ich # Intel, NVidia and other ICH AC'97 Audio +device snd_via8233 # VIA VT8233x Audio + +# MMC/SD +device mmc # MMC/SD bus +device mmcsd # MMC/SD memory card +device sdhci # Generic PCI SD Host Controller + +# VirtIO support +device virtio # Generic VirtIO bus (required) +device virtio_pci # VirtIO PCI device +device vtnet # VirtIO Ethernet device +device virtio_blk # VirtIO Block device +device virtio_scsi # VirtIO SCSI device +device virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers and enchancement support +# NOTE: HYPERV depends on hyperv. They must be added or removed together. +options HYPERV # Hyper-V kernel infrastructure +device hyperv # HyperV drivers + +# Xen HVM Guest Optimizations +# NOTE: XENHVM depends on xenpci. They must be added or removed together. +options XENHVM # Xen HVM kernel infrastructure +device xenpci # Xen HVM Hypervisor services driver + +# VMware support +device vmx # VMware VMXNET3 Ethernet + +# Netmap provides direct access to TX/RX rings on supported NICs +device netmap # netmap(4) support + -- 2.7.0 From cbf19b8d8fc558199bd170bba31d8e12f556bf06 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:30:20 +0200 Subject: [PATCH 002/213] ELECTRO_BLOAT: ElectroBSDify the copy Mainly by removing stuff that isn't relevant for ElectroBSD install media which should only be used to compile the final kernel. --- sys/amd64/conf/ELECTRO_BLOAT | 43 ++++++------------------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index 3d2cc39..e6a7aaa 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -1,25 +1,11 @@ -# -# GENERIC -- Generic kernel configuration file for FreeBSD/amd64 -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ +# ELECTRO_BLOAT -- Modified copy of the GENERIC kernel configuration file +# Used for the release media. cpu HAMMER -ident GENERIC +ident ELECTRO_BLOAT + +# Hacker-movie-compatible colors +options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support @@ -29,7 +15,6 @@ options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols options TCP_OFFLOAD # TCP offload -options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists @@ -84,10 +69,6 @@ options KDB_TRACE # Print a stack trace for a panic. options DDB # Support DDB. options GDB # Support remote GDB. options DEADLKRES # Enable the deadlock resolver -options INVARIANTS # Enable calls of extra sanity checking -options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS -options WITNESS # Enable checks to detect deadlocks and cycles -options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # Make an SMP-capable kernel by default @@ -102,9 +83,6 @@ options ACPI_DMAR device pci options PCI_IOV # PCI SR-IOV support -# Floppy drives -device fdc - # ATA controllers device ahci # AHCI-compatible SATA controllers device ata # Legacy ATA/SATA controllers @@ -121,12 +99,9 @@ options AHD_REG_PRETTY_PRINT # Print register bitfields in debug # output. Adds ~215k to driver. device esp # AMD Am53C974 (Tekram DC-390(T)) device hptiop # Highpoint RocketRaid 3xxx series -device isp # Qlogic family -#device ispfw # Firmware for QLogic HBAs- normally a module device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -#device ncr # NCR/Symbios Logic device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters @@ -169,8 +144,6 @@ device ida # Compaq Smart RAID device mfi # LSI MegaRAID SAS device mlx # Mylex DAC960 family device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s -#XXX pointer/int warnings -#device pst # Promise Supertrak SX6000 device twe # 3ware ATA RAID # NVM Express (NVMe) support @@ -359,7 +332,3 @@ device xenpci # Xen HVM Hypervisor services driver # VMware support device vmx # VMware VMXNET3 Ethernet - -# Netmap provides direct access to TX/RX rings on supported NICs -device netmap # netmap(4) support - -- 2.7.0 From 5361196694231791aaa588e6632ec72d09a29dba Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 23:26:54 +0200 Subject: [PATCH 003/213] ELECTRO_BLOAT: Ditch hpt* drivers which require binary blobs and don't build without firmware --- sys/amd64/conf/ELECTRO_BLOAT | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index e6a7aaa..e3e1973 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -126,10 +126,6 @@ device amr # AMI MegaRAID device arcmsr # Areca SATA II RAID device ciss # Compaq Smart RAID 5* device dpt # DPT Smartcache III, IV - See NOTES for options -device hptmv # Highpoint RocketRAID 182x -device hptnr # Highpoint DC7280, R750 -device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx -device hpt27xx # Highpoint RocketRAID 27xx device iir # Intel Integrated RAID device ips # IBM (Adaptec) ServeRAID device mly # Mylex AcceleRAID/eXtremeRAID -- 2.7.0 From b249198bf7d1a66eff44151df29587efaa1b532b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 12 Oct 2015 18:05:10 +0200 Subject: [PATCH 004/213] ELECTRO_BLOAT/amd64: Polish a bit and import stuff from ELECTRO_BEER. Squash --- sys/amd64/conf/ELECTRO_BLOAT | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index e3e1973..f3eb4c8 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -1,12 +1,26 @@ # ELECTRO_BLOAT -- Modified copy of the GENERIC kernel configuration file # Used for the release media. -cpu HAMMER ident ELECTRO_BLOAT -# Hacker-movie-compatible colors +# One of ElectroBSD's most important features: +# hacker-movie-compatible colors by default! options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) +# Add HTTP accept filter support. The "performance gains" might +# be dubious, but adding it results in nicer logs for applications +# that use it (because requests are less intangled). +options ACCEPT_FILTER_HTTP + +# Build pf into the kernel. It doesn't hurt and supposedly +# works around various bugs that only affect the module build. +device pf + +############################################################################## +# Everything below comes from GENERIC, but "offending" lines have been removed +############################################################################## +cpu HAMMER + makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support -- 2.7.0 From 5793393dfe5ef7453f031947f6ac9427051fb71d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:38:09 +0200 Subject: [PATCH 005/213] Change amd64 default KERNCONF to ELECTRO_BLOAT --- Makefile.inc1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index 6ee07b1..4054393 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1120,6 +1120,8 @@ KERNCONF=${KERNFAST} .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 +.elif ${TARGET_ARCH} == "amd64" +KERNCONF?= ELECTRO_BLOAT .else KERNCONF?= GENERIC .endif -- 2.7.0 From 1ceb0bd794b61c1f90da204336e254c56a6aef94 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Mar 2013 11:28:27 +0100 Subject: [PATCH 006/213] Import ELECTRO_BEER ... from cvsup based git repository at 64cfedadf81b7b6f99. --- sys/amd64/conf/ELECTRO_BEER | 271 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 sys/amd64/conf/ELECTRO_BEER diff --git a/sys/amd64/conf/ELECTRO_BEER b/sys/amd64/conf/ELECTRO_BEER new file mode 100644 index 0000000..31d9c59 --- /dev/null +++ b/sys/amd64/conf/ELECTRO_BEER @@ -0,0 +1,271 @@ +include ELECTRO_BLOAT + +ident ELECTRO_BEER + +device pf + +device ada + +options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) + +options ACCEPT_FILTER_HTTP +############### + +# Debugging for use in -current +nooptions INVARIANTS # Enable calls of extra sanity checking +nooptions INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +nooptions WITNESS # Enable checks to detect deadlocks and cycles +nooptions WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +nooptions FLOWTABLE # per-cpu routing cache +nooptions XENHVM # Include Xen support + +nooptions SCTP # Stream Control Transmission Protocol + +nodevice fdc + +nodevice ataraid # ATA RAID drives +nodevice atapifd # ATAPI floppy drives +nodevice atapist # ATAPI tape drives + +# ATA controllers +nodevice mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +nodevice siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +nodevice ahc # AHA2940 and onboard AIC7xxx devices +nooptions AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +nodevice amd # AMD 53C974 (Tekram DC-390(T)) +nodevice esp # AMD Am53C974 (Tekram DC-390(T)) +nodevice hptiop # Highpoint RocketRaid 3xxx series +nodevice isp # Qlogic family +nodevice ispfw # Firmware for QLogic HBAs- normally a module +nodevice mpt # LSI-Logic MPT-Fusion +nodevice mps # LSI-Logic MPT-Fusion 2 +nodevice ncr # NCR/Symbios Logic +nodevice sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +nodevice trm # Tekram DC395U/UW/F DC315U adapters + +nodevice adv # Advansys SCSI adapters +nodevice adw # Advansys wide SCSI adapters +nodevice aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +nodevice bt # Buslogic/Mylex MultiMaster SCSI adapters + +# RAID controllers interfaced to the SCSI subsystem +nodevice amr # AMI MegaRAID +nodevice arcmsr # Areca SATA II RAID +#XXX it is not 64-bit clean, -scottl +nodevice asr # DPT SmartRAID V, VI and Adaptec SCSI RAID +nodevice ciss # Compaq Smart RAID 5* +nodevice dpt # DPT Smartcache III, IV - See NOTES for options +nodevice hptmv # Highpoint RocketRAID 182x +nodevice hptnr # Highpoint DC7280, R750 +nodevice hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +nodevice hpt27xx # Highpoint RocketRAID 27xx + +nodevice iir # Intel Integrated RAID +nodevice ips # IBM (Adaptec) ServeRAID +nodevice mly # Mylex AcceleRAID/eXtremeRAID +nodevice twa # 3ware 9000 series PATA/SATA RAID +nodevice tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +nodevice aac # Adaptec FSA RAID +nodevice aacraid # Adaptec by PMC RAID +nodevice aacp # SCSI passthrough for aac (requires CAM) +nodevice ida # Compaq Smart RAID +nodevice mfi # LSI MegaRAID SAS +nodevice mlx # Mylex DAC960 family +#XXX pointer/int warnings +nodevice pst # Promise Supertrak SX6000 +nodevice twe # 3ware ATA RAID + + +# Parallel port +nodevice ppc +nodevice ppbus # Parallel port bus (required) +nodevice lpt # Printer +nodevice plip # TCP/IP over parallel +nodevice ppi # Parallel port interface device +nodevice vpo # Requires scbus and da + +# If you've got a "dumb" serial or parallel PCI card that is +# supported by the puc(4) glue driver, uncomment the following +# line to enable it (connects to sio, uart and/or ppc drivers): +nodevice puc + +# PCI Ethernet NICs. +nodevice bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +nodevice de # DEC/Intel DC21x4x (``Tulip'') +nodevice em # Intel PRO/1000 Gigabit Ethernet Family +nodevice igb # Intel PRO/1000 PCIE Server Gigabit Family +nodevice ixgbe # Intel PRO/10GbE PCIE Ethernet Family +nodevice le # AMD Am7900 LANCE and Am79C9xx PCnet +nodevice ti # Alteon Networks Tigon I/II gigabit Ethernet +nodevice txp # 3Com 3cR990 (``Typhoon'') +nodevice vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! + +nodevice ae # Attansic/Atheros L2 FastEthernet +nodevice age # Attansic/Atheros L1 Gigabit Ethernet +nodevice alc # Atheros AR8131/AR8132 Ethernet +nodevice ale # Atheros AR8121/AR8113/AR8114 Ethernet +nodevice bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +nodevice bfe # Broadcom BCM440x 10/100 Ethernet +nodevice cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +nodevice dc # DEC/Intel 21143 and various workalikes +nodevice et # Agere ET1310 10/100/Gigabit Ethernet +nodevice fxp # Intel EtherExpress PRO/100B (82557, 82558) +nodevice gem # Sun GEM/Sun ERI/Apple GMAC +nodevice hme # Sun HME (Happy Meal Ethernet) +nodevice jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +nodevice lge # Level 1 LXT1001 gigabit Ethernet +nodevice msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +nodevice nfe # nVidia nForce MCP on-board Ethernet +nodevice nge # NatSemi DP83820 gigabit Ethernet +nodevice nve # nVidia nForce MCP on-board Ethernet Networking +nodevice pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +nodevice re # RealTek 8139C+/8169/8169S/8110S +nodevice rl # RealTek 8129/8139 +nodevice sf # Adaptec AIC-6915 (``Starfire'') +nodevice sge # Silicon Integrated Systems SiS190/191 +nodevice sis # Silicon Integrated Systems SiS 900/SiS 7016 +nodevice sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +nodevice ste # Sundance ST201 (D-Link DFE-550TX) +nodevice stge # Sundance/Tamarack TC9021 gigabit Ethernet +nodevice tl # Texas Instruments ThunderLAN +nodevice tx # SMC EtherPower II (83c170 ``EPIC'') +nodevice vge # VIA VT612x gigabit Ethernet +nodevice vr # VIA Rhine, Rhine II +nodevice wb # Winbond W89C840F +nodevice xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# ISA Ethernet NICs. pccard NICs included. +nodevice cs # Crystal Semiconductor CS89x0 NIC +# 'device ed' requires 'device miibus' +nodevice ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards +nodevice ex # Intel EtherExpress Pro/10 and Pro/10+ +nodevice ep # Etherlink III based cards +nodevice fe # Fujitsu MB8696x based cards +nodevice sn # SMC's 9000 series of Ethernet chips +nodevice xe # Xircom pccard Ethernet + +nodevice zyd # ZyDAS zb1211/zb1211b wireless NICs +nodevice urio # Diamond Rio 500 MP3 player + +# Wireless NIC cards +nodevice an # Aironet 4500/4800 802.11 wireless NICs. +nodevice ath # Atheros NIC's +nodevice ath_pci # Atheros pci/cardbus glue +nodevice ath_hal # pci/cardbus chip support +nodevice ath_rate_sample # SampleRate tx rate control for ath +nodevice bwi # Broadcom BCM430x/BCM431x wireless NICs. +nodevice bwn # Broadcom BCM43xx wireless NICs. +nodevice ipw # Intel 2100 wireless NICs. +nodevice iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +nodevice iwn # Intel 4965/1000/5000/6000 wireless NICs. +nodevice malo # Marvell Libertas wireless NICs. +nodevice mwl # Marvell 88W8363 802.11n wireless NICs. +nodevice ral # Ralink Technology RT2500 wireless NICs. +nodevice wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +nodevice wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +nodevice padlock_rng # VIA Padlock RNG +nodevice rdrand_rng # Intel Bull Mountain RNG +nodevice gif # IPv6 and IPv4 tunneling +nodevice faith # IPv6-to-IPv4 relaying (translation) + +# USB Serial devices +nodevice uark # Technologies ARK3116 based serial adapters +nodevice ubsa # Belkin F5U103 and compatible serial adapters +nodevice uftdi # For FTDI usb serial adapters +nodevice uipaq # Some WinCE based devices +nodevice uplcom # Prolific PL-2303 serial adapters +nodevice uslcom # SI Labs CP2101/CP2102 serial adapters +nodevice uvisor # Visor and Palm devices +nodevice uvscom # USB serial support for DDI pocket's PHS +# USB Ethernet, requires miibus +nodevice aue # ADMtek USB Ethernet +nodevice axe # ASIX Electronics USB Ethernet +nodevice cdce # Generic USB over Ethernet +nodevice cue # CATC USB Ethernet +nodevice kue # Kawasaki LSI USB Ethernet +nodevice rue # RealTek RTL8150 USB Ethernet +nodevice udav # Davicom DM9601E USB + +# USB support + +nodevice uhci # UHCI PCI->USB interface +nodevice ohci # OHCI PCI->USB interface +nodevice ehci # EHCI PCI->USB interface (USB 2.0) +nodevice xhci # XHCI PCI->USB interface (USB 3.0) +nodevice usb # USB Bus (required) +nodevice ukbd # Keyboard +nodevice umass # Disks/Mass storage - Requires scbus and da + +# FireWire support +nodevice firewire # FireWire bus code +nodevice sbp # SCSI over FireWire (Requires scbus and da) +nodevice fwe # Ethernet over FireWire (non-standard!) +nodevice fwip # IP over FireWire (RFC 2734,3146) +nodevice dcons # Dumb console driver +nodevice dcons_crom # Configuration ROM for dcons + +# Sound support +nodevice snd_es137x # Ensoniq AudioPCI ES137x +nodevice snd_ich # Intel, NVidia and other ICH AC'97 Audio +nodevice snd_uaudio # USB Audio +nodevice snd_via8233 # VIA VT8233x Audio +nodevice snd_cmi # CMedia CMI8338/CMI8738 +nodevice snd_csa # Crystal Semiconductor CS461x/428x +nodevice snd_emu10kx # Creative SoundBlaster Live! and Audigy + +# VirtIO support +nodevice virtio # Generic VirtIO bus (required) +nodevice virtio_pci # VirtIO PCI device +nodevice vtnet # VirtIO Ethernet device +nodevice virtio_blk # VirtIO Block device +nodevice virtio_scsi # VirtIO SCSI device +nodevice virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers +nodevice hyperv # HyperV drivers + +# Xen support +nodevice xenpci # Generic Xen bus + +# VMware support +nodevice vmx # VMware VMXNET3 Ethernet + +# Useful? Taken from http://serverfault.com/questions/64356/freebsd-performance-tuning-sysctls-loader-conf-kernel + +# More scroll space +options SC_HISTORY_SIZE=8192 + +# Same for Intel processors +device coretemp + +# man 4 cpuctl +device cpuctl # CPU control pseudo-device + +# UTF-8 in console (8.x+) +options TEKEN_UTF8 + +nodevice netmap + +# Disabling them prevents ZFS from being loaded. +# Should be bisected. +#nooptions NFS_ROOT +#nooptions NFSLOCKD +#nooptions NFSD +#nooptions NFSCL +nodevice ahd +nodevice mpr +nodevice isci +nodevice ses +nodevice mrsas +nodevice ixl +nodevice ixlv -- 2.7.0 From 05e52ac2be6f9c343ee590d85be1fb5bea3f1123 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 14 Oct 2015 13:19:53 +0200 Subject: [PATCH 007/213] ELECTRO_BEER amd64: Remove lines that now are in ELECTRO_BLOAT. Squash --- sys/amd64/conf/ELECTRO_BEER | 9 --------- 1 file changed, 9 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BEER b/sys/amd64/conf/ELECTRO_BEER index 31d9c59..306621a 100644 --- a/sys/amd64/conf/ELECTRO_BEER +++ b/sys/amd64/conf/ELECTRO_BEER @@ -2,15 +2,6 @@ include ELECTRO_BLOAT ident ELECTRO_BEER -device pf - -device ada - -options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) - -options ACCEPT_FILTER_HTTP -############### - # Debugging for use in -current nooptions INVARIANTS # Enable calls of extra sanity checking nooptions INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS -- 2.7.0 From 28c993f89c409f5d934027c4266bbd8d9a0dac98 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 14 Apr 2015 17:43:38 +0200 Subject: [PATCH 008/213] Add ELECTRO_BEER for i386 --- sys/i386/conf/ELECTRO_BEER | 1 + 1 file changed, 1 insertion(+) create mode 120000 sys/i386/conf/ELECTRO_BEER diff --git a/sys/i386/conf/ELECTRO_BEER b/sys/i386/conf/ELECTRO_BEER new file mode 120000 index 0000000..4a483bb --- /dev/null +++ b/sys/i386/conf/ELECTRO_BEER @@ -0,0 +1 @@ +../../amd64/conf/ELECTRO_BEER \ No newline at end of file -- 2.7.0 From 9691947a509383cbe5117286e531a2131259e678 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 17:06:35 +0200 Subject: [PATCH 009/213] i386: Copy GENERIC to ELECTRO_BLOAT --- Makefile.inc1 | 2 +- sys/i386/conf/ELECTRO_BLOAT | 383 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 sys/i386/conf/ELECTRO_BLOAT diff --git a/Makefile.inc1 b/Makefile.inc1 index 4054393..2359c75 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1120,7 +1120,7 @@ KERNCONF=${KERNFAST} .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 -.elif ${TARGET_ARCH} == "amd64" +.elif ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" KERNCONF?= ELECTRO_BLOAT .else KERNCONF?= GENERIC diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT new file mode 100644 index 0000000..68ac2de --- /dev/null +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -0,0 +1,383 @@ +# +# GENERIC -- Generic kernel configuration file for FreeBSD/i386 +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +cpu I486_CPU +cpu I586_CPU +cpu I686_CPU +ident GENERIC + +makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols +makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support + +options SCHED_ULE # ULE scheduler +options PREEMPTION # Enable kernel thread preemption +options INET # InterNETworking +options INET6 # IPv6 communications protocols +options TCP_OFFLOAD # TCP offload +options SCTP # Stream Control Transmission Protocol +options FFS # Berkeley Fast Filesystem +options SOFTUPDATES # Enable FFS soft updates support +options UFS_ACL # Support for access control lists +options UFS_DIRHASH # Improve performance on big directories +options UFS_GJOURNAL # Enable gjournal-based UFS journaling +options QUOTA # Enable disk quotas for UFS +options MD_ROOT # MD is a potential root device +options NFSCL # Network Filesystem Client +options NFSD # Network Filesystem Server +options NFSLOCKD # Network Lock Manager +options NFS_ROOT # NFS usable as /, requires NFSCL +options MSDOSFS # MSDOS Filesystem +options CD9660 # ISO 9660 Filesystem +options PROCFS # Process filesystem (requires PSEUDOFS) +options PSEUDOFS # Pseudo-filesystem framework +options GEOM_PART_GPT # GUID Partition Tables. +options GEOM_RAID # Soft RAID functionality. +options GEOM_LABEL # Provides labelization +options COMPAT_FREEBSD4 # Compatible with FreeBSD4 +options COMPAT_FREEBSD5 # Compatible with FreeBSD5 +options COMPAT_FREEBSD6 # Compatible with FreeBSD6 +options COMPAT_FREEBSD7 # Compatible with FreeBSD7 +options COMPAT_FREEBSD9 # Compatible with FreeBSD9 +options COMPAT_FREEBSD10 # Compatible with FreeBSD10 +options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI +options KTRACE # ktrace(1) support +options STACK # stack(9) support +options SYSVSHM # SYSV-style shared memory +options SYSVMSG # SYSV-style message queues +options SYSVSEM # SYSV-style semaphores +options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions +options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. +options KBD_INSTALL_CDEV # install a CDEV entry in /dev +options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) +options AUDIT # Security event auditing +options CAPABILITY_MODE # Capsicum capability mode +options CAPABILITIES # Capsicum capabilities +options MAC # TrustedBSD MAC Framework +options KDTRACE_HOOKS # Kernel DTrace hooks +options DDB_CTF # Kernel ELF linker loads CTF data +options INCLUDE_CONFIG_FILE # Include this file in kernel +options RACCT # Resource accounting framework +options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default +options RCTL # Resource limits + +# Debugging support. Always need this: +options KDB # Enable kernel debugger support. +options KDB_TRACE # Print a stack trace for a panic. +# For full debugger support use (turn off in stable branch): +options DDB # Support DDB. +options GDB # Support remote GDB. +options DEADLKRES # Enable the deadlock resolver +options INVARIANTS # Enable calls of extra sanity checking +options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +options WITNESS # Enable checks to detect deadlocks and cycles +options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones + +# To make an SMP kernel, the next two lines are needed +options SMP # Symmetric MultiProcessor Kernel +device apic # I/O APIC + +# CPU frequency control +device cpufreq + +# Bus support. +device acpi +device pci +options PCI_IOV # PCI SR-IOV support + +# Floppy drives +device fdc + +# ATA controllers +device ahci # AHCI-compatible SATA controllers +device ata # Legacy ATA/SATA controllers +options ATA_STATIC_ID # Static device numbering +device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +device ahc # AHA2940 and onboard AIC7xxx devices +options AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +device ahd # AHA39320/29320 and onboard AIC79xx devices +options AHD_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~215k to driver. +device esp # AMD Am53C974 (Tekram DC-390(T)) +device hptiop # Highpoint RocketRaid 3xxx series +device isp # Qlogic family +#device ispfw # Firmware for QLogic HBAs- normally a module +device mpt # LSI-Logic MPT-Fusion +device mps # LSI-Logic MPT-Fusion 2 +device mpr # LSI-Logic MPT-Fusion 3 +#device ncr # NCR/Symbios Logic +device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +device trm # Tekram DC395U/UW/F DC315U adapters + +device adv # Advansys SCSI adapters +device adw # Advansys wide SCSI adapters +device aha # Adaptec 154x SCSI adapters +device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +device bt # Buslogic/Mylex MultiMaster SCSI adapters + +device ncv # NCR 53C500 +device nsp # Workbit Ninja SCSI-3 +device stg # TMC 18C30/18C50 +device isci # Intel C600 SAS controller + +# ATA/SCSI peripherals +device scbus # SCSI bus (required for ATA/SCSI) +device ch # SCSI media changers +device da # Direct Access (disks) +device sa # Sequential Access (tape etc) +device cd # CD +device pass # Passthrough device (direct ATA/SCSI access) +device ses # Enclosure Services (SES and SAF-TE) +#device ctl # CAM Target Layer + +# RAID controllers interfaced to the SCSI subsystem +device amr # AMI MegaRAID +device arcmsr # Areca SATA II RAID +device ciss # Compaq Smart RAID 5* +device dpt # DPT Smartcache III, IV - See NOTES for options +device hptmv # Highpoint RocketRAID 182x +device hptnr # Highpoint DC7280, R750 +device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +device hpt27xx # Highpoint RocketRAID 27xx +device iir # Intel Integrated RAID +device ips # IBM (Adaptec) ServeRAID +device mly # Mylex AcceleRAID/eXtremeRAID +device twa # 3ware 9000 series PATA/SATA RAID +device tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +device aac # Adaptec FSA RAID +device aacp # SCSI passthrough for aac (requires CAM) +device aacraid # Adaptec by PMC RAID +device ida # Compaq Smart RAID +device mfi # LSI MegaRAID SAS +device mlx # Mylex DAC960 family +device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s +device pst # Promise Supertrak SX6000 +device twe # 3ware ATA RAID + +# NVM Express (NVMe) support +device nvme # base NVMe driver +device nvd # expose NVMe namespace as disks, depends on nvme + +# atkbdc0 controls both the keyboard and the PS/2 mouse +device atkbdc # AT keyboard controller +device atkbd # AT keyboard +device psm # PS/2 mouse + +device kbdmux # keyboard multiplexer + +device vga # VGA video card driver +options VESA # Add support for VESA BIOS Extensions (VBE) + +device splash # Splash screen and screen saver support + +# syscons is the default console driver, resembling an SCO console +device sc +options SC_PIXEL_MODE # add support for the raster text mode + +# vt is the new video console driver +device vt +device vt_vga + +device agp # support several AGP chipsets + +# Power management support (see NOTES for more options) +#device apm +# Add suspend/resume support for the i8254. +device pmtimer + +# PCCARD (PCMCIA) support +# PCMCIA and cardbus bridge support +device cbb # cardbus (yenta) bridge +device pccard # PC Card (16-bit) bus +device cardbus # CardBus (32-bit) bus + +# Serial (COM) ports +device uart # Generic UART driver + +# Parallel port +device ppc +device ppbus # Parallel port bus (required) +device lpt # Printer +device ppi # Parallel port interface device +#device vpo # Requires scbus and da + +device puc # Multi I/O cards and multi-channel UARTs + +# PCI Ethernet NICs. +device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +device de # DEC/Intel DC21x4x (``Tulip'') +device em # Intel PRO/1000 Gigabit Ethernet Family +device igb # Intel PRO/1000 PCIE Server Gigabit Family +device ixgb # Intel PRO/10GbE Ethernet Card +device le # AMD Am7900 LANCE and Am79C9xx PCnet +device ti # Alteon Networks Tigon I/II gigabit Ethernet +device txp # 3Com 3cR990 (``Typhoon'') +device vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! +device miibus # MII bus support +device ae # Attansic/Atheros L2 FastEthernet +device age # Attansic/Atheros L1 Gigabit Ethernet +device alc # Atheros AR8131/AR8132 Ethernet +device ale # Atheros AR8121/AR8113/AR8114 Ethernet +device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +device bfe # Broadcom BCM440x 10/100 Ethernet +device bge # Broadcom BCM570xx Gigabit Ethernet +device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +device dc # DEC/Intel 21143 and various workalikes +device et # Agere ET1310 10/100/Gigabit Ethernet +device fxp # Intel EtherExpress PRO/100B (82557, 82558) +device gem # Sun GEM/Sun ERI/Apple GMAC +device hme # Sun HME (Happy Meal Ethernet) +device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +device lge # Level 1 LXT1001 gigabit Ethernet +device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +device nfe # nVidia nForce MCP on-board Ethernet +device nge # NatSemi DP83820 gigabit Ethernet +device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +device re # RealTek 8139C+/8169/8169S/8110S +device rl # RealTek 8129/8139 +device sf # Adaptec AIC-6915 (``Starfire'') +device sge # Silicon Integrated Systems SiS190/191 +device sis # Silicon Integrated Systems SiS 900/SiS 7016 +device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +device ste # Sundance ST201 (D-Link DFE-550TX) +device stge # Sundance/Tamarack TC9021 gigabit Ethernet +device tl # Texas Instruments ThunderLAN +device tx # SMC EtherPower II (83c170 ``EPIC'') +device vge # VIA VT612x gigabit Ethernet +device vr # VIA Rhine, Rhine II +device vte # DM&P Vortex86 RDC R6040 Fast Ethernet +device wb # Winbond W89C840F +device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# ISA Ethernet NICs. pccard NICs included. +device cs # Crystal Semiconductor CS89x0 NIC +# 'device ed' requires 'device miibus' +device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards +device ex # Intel EtherExpress Pro/10 and Pro/10+ +device ep # Etherlink III based cards +device fe # Fujitsu MB8696x based cards +device ie # EtherExpress 8/16, 3C507, StarLAN 10 etc. +device sn # SMC's 9000 series of Ethernet chips +device xe # Xircom pccard Ethernet + +# Wireless NIC cards +device wlan # 802.11 support +options IEEE80211_DEBUG # enable debug msgs +options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's +options IEEE80211_SUPPORT_MESH # enable 802.11s draft support +device wlan_wep # 802.11 WEP support +device wlan_ccmp # 802.11 CCMP support +device wlan_tkip # 802.11 TKIP support +device wlan_amrr # AMRR transmit rate control algorithm +device an # Aironet 4500/4800 802.11 wireless NICs. +device ath # Atheros NICs +device ath_pci # Atheros pci/cardbus glue +device ath_hal # pci/cardbus chip support +options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors +options AH_AR5416_INTERRUPT_MITIGATION # AR5416 interrupt mitigation +options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later +device ath_rate_sample # SampleRate tx rate control for ath +#device bwi # Broadcom BCM430x/BCM431x wireless NICs. +#device bwn # Broadcom BCM43xx wireless NICs. +device ipw # Intel 2100 wireless NICs. +device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +device iwn # Intel 4965/1000/5000/6000 wireless NICs. +device malo # Marvell Libertas wireless NICs. +device mwl # Marvell 88W8363 802.11n wireless NICs. +device ral # Ralink Technology RT2500 wireless NICs. +device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +#device wl # Older non 802.11 Wavelan wireless NIC. +device wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +device loop # Network loopback +device random # Entropy device +device padlock_rng # VIA Padlock RNG +device rdrand_rng # Intel Bull Mountain RNG +device ether # Ethernet support +device vlan # 802.1Q VLAN support +device tun # Packet tunnel. +device md # Memory "disks" +device gif # IPv6 and IPv4 tunneling +device firmware # firmware assist module + +# The `bpf' device enables the Berkeley Packet Filter. +# Be aware of the administrative consequences of enabling this! +# Note that 'bpf' is required for DHCP. +device bpf # Berkeley packet filter + +# USB support +options USB_DEBUG # enable debug msgs +device uhci # UHCI PCI->USB interface +device ohci # OHCI PCI->USB interface +device ehci # EHCI PCI->USB interface (USB 2.0) +device xhci # XHCI PCI->USB interface (USB 3.0) +device usb # USB Bus (required) +device ukbd # Keyboard +device umass # Disks/Mass storage - Requires scbus and da + +# Sound support +device sound # Generic sound driver (required) +device snd_cmi # CMedia CMI8338/CMI8738 +device snd_csa # Crystal Semiconductor CS461x/428x +device snd_emu10kx # Creative SoundBlaster Live! and Audigy +device snd_es137x # Ensoniq AudioPCI ES137x +device snd_hda # Intel High Definition Audio +device snd_ich # Intel, NVidia and other ICH AC'97 Audio +device snd_via8233 # VIA VT8233x Audio + +# MMC/SD +device mmc # MMC/SD bus +device mmcsd # MMC/SD memory card +device sdhci # Generic PCI SD Host Controller + +# VirtIO support +device virtio # Generic VirtIO bus (required) +device virtio_pci # VirtIO PCI device +device vtnet # VirtIO Ethernet device +device virtio_blk # VirtIO Block device +device virtio_scsi # VirtIO SCSI device +device virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers and enchancement support +# NOTE: HYPERV depends on hyperv. They must be added or removed together. +options HYPERV # Hyper-V kernel infrastructure +device hyperv # HyperV drivers + +# Xen HVM Guest Optimizations +# NOTE: XENHVM depends on xenpci. They must be added or removed together. +options XENHVM # Xen HVM kernel infrastructure +device xenpci # Xen HVM Hypervisor services driver + +# VMware support +device vmx # VMware VMXNET3 Ethernet + +# Required for ZFS when compiled with clang. +# For details see UPDATING entry 20121223. +options KSTACK_PAGES=4 -- 2.7.0 From e4fc63c50445d7ab87f1aca074c084cb7c3951ab Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 17:55:42 +0200 Subject: [PATCH 010/213] Adjust shiny new ELECTRO_BLOAT i386 for ElectroBSD --- sys/i386/conf/ELECTRO_BLOAT | 60 +++++++++++---------------------------------- 1 file changed, 14 insertions(+), 46 deletions(-) diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT index 68ac2de..4a5dbee 100644 --- a/sys/i386/conf/ELECTRO_BLOAT +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -1,27 +1,11 @@ -# -# GENERIC -- Generic kernel configuration file for FreeBSD/i386 -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ - -cpu I486_CPU -cpu I586_CPU +# ELECTRO_BLOAT -- Modified copy of the GENERIC kernel configuration file +# Used for the release media. + cpu I686_CPU -ident GENERIC +ident ELECTRO_BLOAT + +# Hacker-movie-compatible colors +options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support @@ -31,7 +15,6 @@ options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols options TCP_OFFLOAD # TCP offload -options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists @@ -48,12 +31,7 @@ options CD9660 # ISO 9660 Filesystem options PROCFS # Process filesystem (requires PSEUDOFS) options PSEUDOFS # Pseudo-filesystem framework options GEOM_PART_GPT # GUID Partition Tables. -options GEOM_RAID # Soft RAID functionality. options GEOM_LABEL # Provides labelization -options COMPAT_FREEBSD4 # Compatible with FreeBSD4 -options COMPAT_FREEBSD5 # Compatible with FreeBSD5 -options COMPAT_FREEBSD6 # Compatible with FreeBSD6 -options COMPAT_FREEBSD7 # Compatible with FreeBSD7 options COMPAT_FREEBSD9 # Compatible with FreeBSD9 options COMPAT_FREEBSD10 # Compatible with FreeBSD10 options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI @@ -84,10 +62,6 @@ options KDB_TRACE # Print a stack trace for a panic. options DDB # Support DDB. options GDB # Support remote GDB. options DEADLKRES # Enable the deadlock resolver -options INVARIANTS # Enable calls of extra sanity checking -options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS -options WITNESS # Enable checks to detect deadlocks and cycles -options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # To make an SMP kernel, the next two lines are needed @@ -102,9 +76,6 @@ device acpi device pci options PCI_IOV # PCI SR-IOV support -# Floppy drives -device fdc - # ATA controllers device ahci # AHCI-compatible SATA controllers device ata # Legacy ATA/SATA controllers @@ -122,11 +93,9 @@ options AHD_REG_PRETTY_PRINT # Print register bitfields in debug device esp # AMD Am53C974 (Tekram DC-390(T)) device hptiop # Highpoint RocketRaid 3xxx series device isp # Qlogic family -#device ispfw # Firmware for QLogic HBAs- normally a module device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -#device ncr # NCR/Symbios Logic device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters @@ -177,10 +146,6 @@ device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s device pst # Promise Supertrak SX6000 device twe # 3ware ATA RAID -# NVM Express (NVMe) support -device nvme # base NVMe driver -device nvd # expose NVMe namespace as disks, depends on nvme - # atkbdc0 controls both the keyboard and the PS/2 mouse device atkbdc # AT keyboard controller device atkbd # AT keyboard @@ -203,8 +168,6 @@ device vt_vga device agp # support several AGP chipsets -# Power management support (see NOTES for more options) -#device apm # Add suspend/resume support for the i8254. device pmtimer @@ -378,6 +341,11 @@ device xenpci # Xen HVM Hypervisor services driver # VMware support device vmx # VMware VMXNET3 Ethernet -# Required for ZFS when compiled with clang. -# For details see UPDATING entry 20121223. +# This used to be required for ZFS when compiled with clang. +# For details see UPDATING entry 20121223. After r286288 it's +# probably no longer necessary, but for now we keep it anyway. options KSTACK_PAGES=4 + +# Increase the size of the kernel virtual address space +# so ZFS can cache more stuff. +options KVA_PAGES=512 -- 2.7.0 From 46d088a3b71c0bd1c464a5e4d0ff6a839b86b62f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 23:28:17 +0200 Subject: [PATCH 011/213] ELECTRO_BLOAT i386: Ditch hpt* drivers which require binary blobs and don't build without firmware --- sys/i386/conf/ELECTRO_BLOAT | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT index 4a5dbee..c275749 100644 --- a/sys/i386/conf/ELECTRO_BLOAT +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -125,10 +125,6 @@ device amr # AMI MegaRAID device arcmsr # Areca SATA II RAID device ciss # Compaq Smart RAID 5* device dpt # DPT Smartcache III, IV - See NOTES for options -device hptmv # Highpoint RocketRAID 182x -device hptnr # Highpoint DC7280, R750 -device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx -device hpt27xx # Highpoint RocketRAID 27xx device iir # Intel Integrated RAID device ips # IBM (Adaptec) ServeRAID device mly # Mylex AcceleRAID/eXtremeRAID -- 2.7.0 From bf58f93adbea7069f5a112d213890908403dc63e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 20 Aug 2015 16:36:16 +0200 Subject: [PATCH 012/213] ELECTRO_BLOAT (amd64/i386): Remove devices that require proprietary firmware ElectroBSD doesn't ship with --- sys/amd64/conf/ELECTRO_BLOAT | 11 ----------- sys/i386/conf/ELECTRO_BLOAT | 12 ------------ 2 files changed, 23 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index f3eb4c8..ad6ad1a 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -116,7 +116,6 @@ device hptiop # Highpoint RocketRaid 3xxx series device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters device adv # Advansys SCSI adapters @@ -202,7 +201,6 @@ device ppi # Parallel port interface device device puc # Multi I/O cards and multi-channel UARTs # PCI Ethernet NICs. -device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family device igb # Intel PRO/1000 PCIE Server Gigabit Family @@ -222,7 +220,6 @@ device ae # Attansic/Atheros L2 FastEthernet device age # Attansic/Atheros L1 Gigabit Ethernet device alc # Atheros AR8131/AR8132 Ethernet device ale # Atheros AR8121/AR8113/AR8114 Ethernet -device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet device bfe # Broadcom BCM440x 10/100 Ethernet device bge # Broadcom BCM570xx Gigabit Ethernet device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn @@ -271,14 +268,7 @@ options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later device ath_rate_sample # SampleRate tx rate control for ath #device bwi # Broadcom BCM430x/BCM431x wireless NICs. #device bwn # Broadcom BCM43xx wireless NICs. -device ipw # Intel 2100 wireless NICs. -device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. -device iwn # Intel 4965/1000/5000/6000 wireless NICs. device malo # Marvell Libertas wireless NICs. -device mwl # Marvell 88W8363 802.11n wireless NICs. -device ral # Ralink Technology RT2500 wireless NICs. -device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. -device wpi # Intel 3945ABG wireless NICs. # Pseudo devices. device loop # Network loopback @@ -310,7 +300,6 @@ device umass # Disks/Mass storage - Requires scbus and da # Sound support device sound # Generic sound driver (required) device snd_cmi # CMedia CMI8338/CMI8738 -device snd_csa # Crystal Semiconductor CS461x/428x device snd_emu10kx # Creative SoundBlaster Live! and Audigy device snd_es137x # Ensoniq AudioPCI ES137x device snd_hda # Intel High Definition Audio diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT index c275749..d1721f8 100644 --- a/sys/i386/conf/ELECTRO_BLOAT +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -96,7 +96,6 @@ device isp # Qlogic family device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters device adv # Advansys SCSI adapters @@ -186,7 +185,6 @@ device ppi # Parallel port interface device device puc # Multi I/O cards and multi-channel UARTs # PCI Ethernet NICs. -device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family device igb # Intel PRO/1000 PCIE Server Gigabit Family @@ -203,7 +201,6 @@ device ae # Attansic/Atheros L2 FastEthernet device age # Attansic/Atheros L1 Gigabit Ethernet device alc # Atheros AR8131/AR8132 Ethernet device ale # Atheros AR8121/AR8113/AR8114 Ethernet -device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet device bfe # Broadcom BCM440x 10/100 Ethernet device bge # Broadcom BCM570xx Gigabit Ethernet device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn @@ -264,15 +261,7 @@ options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later device ath_rate_sample # SampleRate tx rate control for ath #device bwi # Broadcom BCM430x/BCM431x wireless NICs. #device bwn # Broadcom BCM43xx wireless NICs. -device ipw # Intel 2100 wireless NICs. -device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. -device iwn # Intel 4965/1000/5000/6000 wireless NICs. device malo # Marvell Libertas wireless NICs. -device mwl # Marvell 88W8363 802.11n wireless NICs. -device ral # Ralink Technology RT2500 wireless NICs. -device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. -#device wl # Older non 802.11 Wavelan wireless NIC. -device wpi # Intel 3945ABG wireless NICs. # Pseudo devices. device loop # Network loopback @@ -304,7 +293,6 @@ device umass # Disks/Mass storage - Requires scbus and da # Sound support device sound # Generic sound driver (required) device snd_cmi # CMedia CMI8338/CMI8738 -device snd_csa # Crystal Semiconductor CS461x/428x device snd_emu10kx # Creative SoundBlaster Live! and Audigy device snd_es137x # Ensoniq AudioPCI ES137x device snd_hda # Intel High Definition Audio -- 2.7.0 From 939df5c8ee37a044aca8658eba89fee59a12af90 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 1 Oct 2015 12:17:00 +0200 Subject: [PATCH 013/213] ELECTRO_BLOAT (amd64/i386): Import the 'device crypto' line from GENERIC While we don't need it for IPSEC, having it in the kernel makes upgrading from FreeBSD to ElectroBSD more convenient when using cloudiatr. By default cloudiatr only adds currenty-loaded modules to the bpool, so if crypto.ko is part of the currently running kernel, but the installed kernel requires it as module, the newly-setup system will not boot unless the user adds the module manually. --- sys/amd64/conf/ELECTRO_BLOAT | 3 +++ sys/i386/conf/ELECTRO_BLOAT | 3 +++ 2 files changed, 6 insertions(+) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index ad6ad1a..6840d69 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -331,3 +331,6 @@ device xenpci # Xen HVM Hypervisor services driver # VMware support device vmx # VMware VMXNET3 Ethernet + +# The crypto framework is required by IPSEC +device crypto # Required by IPSEC diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT index d1721f8..7062329 100644 --- a/sys/i386/conf/ELECTRO_BLOAT +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -325,6 +325,9 @@ device xenpci # Xen HVM Hypervisor services driver # VMware support device vmx # VMware VMXNET3 Ethernet +# The crypto framework is required by IPSEC +device crypto # Required by IPSEC + # This used to be required for ZFS when compiled with clang. # For details see UPDATING entry 20121223. After r286288 it's # probably no longer necessary, but for now we keep it anyway. -- 2.7.0 From 03786c618fd48fe0ed45a4bebe3fb04e459423ed Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Aug 2011 19:19:51 +0200 Subject: [PATCH 014/213] Register the product id for Feiya Memory Bar --- sys/dev/usb/usbdevs | 1 + 1 file changed, 1 insertion(+) diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index 2a55e7c..5368f5c 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -1864,6 +1864,7 @@ product FALCOM SAMBA 0x0005 FTDI compatible adapter /* FEIYA products */ product FEIYA DUMMY 0x0000 Dummy product +product FEIYA MEMORY_BAR 0x1000 Memory Bar product FEIYA 5IN1 0x1132 5-in-1 Card Reader product FEIYA ELANGO 0x6200 MicroSDHC Card Reader product FEIYA AC110 0x6300 AC-110 Card Reader -- 2.7.0 From 4e84122dd33406b008cfb567a69b23ae9798bdc3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Aug 2011 19:22:46 +0200 Subject: [PATCH 015/213] Add the full name of 'Feya Technology Corp.' --- sys/dev/usb/usbdevs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index 5368f5c..95dd5d1 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -446,7 +446,7 @@ vendor DIGIANSWER 0x08fd Digianswer vendor AUTHENTEC 0x08ff AuthenTec vendor AUDIOTECHNICA 0x0909 Audio-Technica vendor TRUMPION 0x090a Trumpion Microelectronics -vendor FEIYA 0x090c Feiya +vendor FEIYA 0x090c Feiya Technology Corp. vendor ALATION 0x0910 Alation Systems vendor GLOBESPAN 0x0915 Globespan vendor CONCORDCAMERA 0x0919 Concord Camera -- 2.7.0 From ffd670893a4e10d78abb1406742a51c049fede54 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 7 Mar 2011 12:59:03 +0100 Subject: [PATCH 016/213] In log_sysevent(), only warn about the unsupported type if the type is different than the last unsupported one. --- .../compat/opensolaris/kern/opensolaris_sysevent.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sys/cddl/compat/opensolaris/kern/opensolaris_sysevent.c b/sys/cddl/compat/opensolaris/kern/opensolaris_sysevent.c index c6ae497..08debab 100644 --- a/sys/cddl/compat/opensolaris/kern/opensolaris_sysevent.c +++ b/sys/cddl/compat/opensolaris/kern/opensolaris_sysevent.c @@ -288,11 +288,21 @@ log_sysevent(sysevent_t *evp, int flag, sysevent_id_t *eid) break; } default: -#if 0 - printf("%s: type %d is not implemented\n", __func__, - nvpair_type(elem)); -#endif + { + static int last_unsupported_type; + int unsupported_type = nvpair_type(elem); + + if (last_unsupported_type != unsupported_type) + { + printf("%s: type %d is not implemented\n", + __func__, unsupported_type); + last_unsupported_type = unsupported_type; + if (NULL != ev->se_subclass) + printf("%s: ev->se_subclass is %s\n", + __func__, ev->se_subclass); + } break; + } } } -- 2.7.0 From 202dd423571f5f943c1b18b018409646d91d7ee4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 Feb 2013 14:42:00 +0100 Subject: [PATCH 017/213] Let g_eli_*read_done() deal with lost devices without causing panics Seems to fix kern/162036 for me. --- sys/geom/eli/g_eli.c | 3 ++- sys/geom/eli/g_eli_privacy.c | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index a2b4e65..d1854cb 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -214,7 +214,8 @@ g_eli_read_done(struct bio *bp) pbp->bio_driver2 = NULL; } g_io_deliver(pbp, pbp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); return; } mtx_lock(&sc->sc_queue_mtx); diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c index d636e1f..6ed5846 100644 --- a/sys/geom/eli/g_eli_privacy.c +++ b/sys/geom/eli/g_eli_privacy.c @@ -87,7 +87,8 @@ g_eli_crypto_read_done(struct cryptop *crp) bp->bio_error = crp->crp_etype; } sc = bp->bio_to->geom->softc; - g_eli_key_drop(sc, crp->crp_desc->crd_key); + if (sc != NULL) + g_eli_key_drop(sc, crp->crp_desc->crd_key); /* * Do we have all sectors already? */ @@ -104,7 +105,8 @@ g_eli_crypto_read_done(struct cryptop *crp) * Read is finished, send it up. */ g_io_deliver(bp, bp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); return (0); } -- 2.7.0 From 3785abe2421ca4e0afbc12c6e273b530aba1c0ff Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 Feb 2013 11:09:20 +0100 Subject: [PATCH 018/213] Let g_eli_write_done() verify that sc isn't NULL as well It looks similar enough to g_eli_read_done() to be affected by kern/162036 as well. --- sys/geom/eli/g_eli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index d1854cb..ac5ca7c 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -260,7 +260,8 @@ g_eli_write_done(struct bio *bp) */ sc = pbp->bio_to->geom->softc; g_io_deliver(pbp, pbp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); } /* -- 2.7.0 From 4ab05933bc56ea124599f07c532ae722f011660f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 16 Oct 2013 22:28:16 +0200 Subject: [PATCH 019/213] Disable assertions in llvm and clang for the 10.0 release cycle. --- lib/clang/clang.build.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/clang/clang.build.mk b/lib/clang/clang.build.mk index e90bd3f..b70fa78 100644 --- a/lib/clang/clang.build.mk +++ b/lib/clang/clang.build.mk @@ -8,7 +8,7 @@ CFLAGS+= -I${LLVM_SRCS}/include -I${CLANG_SRCS}/include \ -I${LLVM_SRCS}/${SRCDIR} ${INCDIR:C/^/-I${LLVM_SRCS}\//} -I. \ -I${LLVM_SRCS}/../../lib/clang/include \ -DLLVM_ON_UNIX -DLLVM_ON_FREEBSD \ - -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS #-DNDEBUG + -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DNDEBUG .if ${MK_CLANG_FULL} != "no" CFLAGS+= -DCLANG_ENABLE_ARCMT \ -- 2.7.0 From 0e3383799dd4c1c19c0de2639e5bf67da8f1c4b1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 20 Dec 2013 18:45:00 +0100 Subject: [PATCH 020/213] Let g_eli_keyfiles_load() log the size of the keyfile loaded --- sys/geom/eli/g_eli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index ac5ca7c..2f59b95 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -955,8 +955,8 @@ g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider) name); return (0); } - G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file, - provider, name); + G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s) (size: %d).", file, + provider, name, (unsigned)size); g_eli_crypto_hmac_update(ctx, data, size); } } -- 2.7.0 From bccc4b9a6d0abb4f9c3fb6f78fe0bd5a69844072 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 21 Dec 2013 19:12:53 +0100 Subject: [PATCH 021/213] Log message for the previous commit --- sys/geom/eli/g_eli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 2f59b95..48fa6bb 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -1120,6 +1120,7 @@ g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) g_eli_keyfiles_clear(pp->name); return (NULL); } + G_ELI_DEBUG(0, "Clearing data for keyfiles for %s.", pp->name); g_eli_keyfiles_clear(pp->name); G_ELI_DEBUG(1, "Using Master Key %u for %s.", nkey, pp->name); break; -- 2.7.0 From 670fc511b41ecd8d983ff99d6458c6e136bb3557 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 4 May 2014 23:19:04 +0200 Subject: [PATCH 022/213] newvers.sh: Skip dirty tree detection. Too fucking slow --- sys/conf/newvers.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index a5d4749..fdcb808 100644 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -193,10 +193,10 @@ if [ -n "$git_cmd" ] ; then if [ -n "$git_b" ] ; then git="${git}(${git_b})" fi - if $git_cmd --work-tree=${SYSDIR}/.. diff-index \ - --name-only HEAD | read dummy; then - git="${git}-dirty" - fi +# if $git_cmd --work-tree=${SYSDIR}/.. diff-index \ +# --name-only HEAD | read dummy; then +# git="${git}-dirty" +# fi fi if [ -n "$p4_cmd" ] ; then -- 2.7.0 From d61282ee6b9f705fc0843e93f438dc9139d043f4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 13:27:28 +0100 Subject: [PATCH 023/213] ggated: Ignore SIGPIPE to prevent DoS ... by a single prematurely closed client connection. --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index e234cb5..36a8d52 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1025,6 +1025,7 @@ main(int argc, char *argv[]) pidfile_write(pfh); signal(SIGCHLD, SIG_IGN); + signal(SIGPIPE, SIG_IGN); sfd = socket(AF_INET, SOCK_STREAM, 0); if (sfd == -1) -- 2.7.0 From 5bca7ebd3fe4ca8f1ee0b837ca47d3ed980e1937 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 13:47:31 +0100 Subject: [PATCH 024/213] ggated: Remove connection if the initial packet couldn't be sent Should help to mitigate DoS after flooding ggated with incomplete requests: error: accept(): Software caused connection abort. error: Exiting. --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 36a8d52..7574e20 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -928,6 +928,7 @@ handshake(struct sockaddr *from, int sfd) if (data == -1) { sendfail(sfd, errno, "Error while sending initial packet: %s.", strerror(errno)); + connection_remove(conn); return (0); } -- 2.7.0 From 83d33c2ca41b95bd2124c34b890fc0a9d1aeafab Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 14:09:24 +0100 Subject: [PATCH 025/213] ggated: Continue if accept() is interrupted or the remote connection is lost --- sbin/ggate/ggated/ggated.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 7574e20..fc7c86a 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1050,9 +1050,11 @@ main(int argc, char *argv[]) for (;;) { fromlen = sizeof(from); tmpsfd = accept(sfd, &from, &fromlen); - if (tmpsfd == -1) + if (tmpsfd == -1) { + if (errno == EINTR || errno == ECONNABORTED) + continue; g_gate_xlog("accept(): %s.", strerror(errno)); - + } if (got_sighup) { got_sighup = 0; exports_get(); -- 2.7.0 From ca2434680438e5a5158d29b7a947e85f3bae5635 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 15:18:27 +0100 Subject: [PATCH 026/213] ggated: Initialize conn->c_diskfd in connection_new() --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index fc7c86a..9d52345 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -453,7 +453,7 @@ connection_new(struct g_gate_cinit *cinit, struct sockaddr *s, int sfd) conn->c_token = cinit->gc_token; ip = htonl(((struct sockaddr_in *)(void *)s)->sin_addr.s_addr); conn->c_srcip = ip; - conn->c_sendfd = conn->c_recvfd = -1; + conn->c_sendfd = conn->c_recvfd = conn->c_diskfd = -1; if ((cinit->gc_flags & GGATE_FLAG_SEND) != 0) conn->c_sendfd = sfd; else -- 2.7.0 From 18b3f216207d569949bd18a719bd78a50231ff6b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 14:16:47 +0100 Subject: [PATCH 027/213] ggated: Prevent c_diskfd leaks through connection_remove() Should help against DoS: [...] debug: Connection created [127.0.0.1, /tank/scratch/testfile]. debug: New connection created (token=2197914058). debug: exports[/tank/scratch/testfile2]: Path mismatch. debug: Sending initial packet. error: accept(): Too many open files. error: Exiting. --- sbin/ggate/ggated/ggated.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 9d52345..6d55218 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -512,6 +512,8 @@ connection_remove(struct ggd_connection *conn) close(conn->c_sendfd); if (conn->c_recvfd != -1) close(conn->c_recvfd); + if (conn->c_diskfd != -1) + close(conn->c_diskfd); free(conn->c_path); free(conn); } -- 2.7.0 From fc989a9958adfb4a2a72fc04ceab74a1e0cba272 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 15:52:39 +0100 Subject: [PATCH 028/213] ggated: Check for connection_add() failures properly Prevents a socket leak --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6d55218..6236c50 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -890,7 +890,7 @@ handshake(struct sockaddr *from, int sfd) */ g_gate_log(LOG_DEBUG, "Found existing connection (token=%lu).", (unsigned long)conn->c_token); - if (connection_add(conn, &cinit, from, sfd) == -1) { + if (connection_add(conn, &cinit, from, sfd) == EEXIST) { connection_remove(conn); return (0); } -- 2.7.0 From ffafe2275e35037c589094b95de89334eed0e15f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 8 Dec 2014 17:59:38 +0100 Subject: [PATCH 029/213] ggated: Do not leak stack data in sendfail() --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6236c50..4bb1f7c 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -591,6 +591,7 @@ sendfail(int sfd, int error, const char *fmt, ...) va_list ap; ssize_t data; + bzero(&sinit, sizeof(sinit)); sinit.gs_error = error; g_gate_swap2n_sinit(&sinit); data = g_gate_send(sfd, &sinit, sizeof(sinit), 0); -- 2.7.0 From 84491ae5a71d1925a4e4f1875ca9d3664920bd08 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 15:24:58 +0200 Subject: [PATCH 030/213] ggated recv_thread(): Do not queue incomplete WRITE requests Verifying that g_gate_recv()'s return code isn't -1 is insufficient as it's a thin wrapper arround recv(2) which, quoting its man page, "may still return less data than requested if a signal is caught, an error or disconnect occurs, or the next data to be received is of a different type than that returned". Previously incomplete WRITE requests would be scheduled with partially uninitialized memory, potentially resulting in file system corruption or, worse, bogus data being later on returned as valid. Security impact: A MITM may cause data corruption by disrupting the connection from ggatec's send_thread() to ggated's recv_thread() at the right point in time. This does not require access to the plain text traffic but if encryption is involved the attacker would have to guess that it's ggate traffic and disrupt connections blindly, hoping that some of the disruptions trigger the bug. The issue was discovered after ZFS on the ggatec side reported checksum errors which weren't reproducible on the ggated side where ZFS had received and checksummed bogus data. The ggate traffic was tunneled through SSH and Tor with sshd running as Tor location hidden service. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 4bb1f7c..9883355 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -669,6 +669,9 @@ recv_thread(void *arg) if (data == -1) { g_gate_xlog("Error while receiving data: %s.", strerror(errno)); + } else if ((uint32_t)data != req->r_length) { + g_gate_xlog("Received %d bytes of data while " + "expecting %u.", data, req->r_length); } } -- 2.7.0 From 9adc7710a0e5d446212e3e47eaedd0c21e5af9ac Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 12:09:40 +0200 Subject: [PATCH 031/213] ggated recv_thread(): Do not queue requests with invalid values ... that would cause abort()s when read by the disk_thread() later on. From ggatec's point of view it doesn't make a difference as the connection will get closed either way, but at least the admin on the server side doesn't have to deal with core dumps. Security impact: An authenticated attacker may intentionally cause the ggated process that handles the attacker's connection to core dump and thus use more disk space than intentionally provisioned by the server admin. Without the following patch ggated core dumps may require more than 100 GB of disk space. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 9883355..6b53dec 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -654,6 +654,23 @@ recv_thread(void *arg) (intmax_t)req->r_offset, (unsigned)req->r_length); /* + * Reject requests that violate assertions in disk_thread(). + */ + if (req->r_cmd != GGATE_CMD_READ && + req->r_cmd != GGATE_CMD_WRITE) { + g_gate_xlog("Request contains invalid command."); + } + if (req->r_offset + req->r_length > + (uintmax_t)conn->c_mediasize) { + g_gate_xlog("Request out of bounds."); + } + if (req->r_offset % conn->c_sectorsize != 0 || + req->r_length % conn->c_sectorsize != 0) { + g_gate_xlog("Request length or offset does " + "not fit sector size."); + } + + /* * Allocate memory for data. */ req->r_data = malloc_waitok(req->r_length); -- 2.7.0 From 39bcaf027a53d87725ac54e1ee415ecd21e41abe Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 19:52:54 +0200 Subject: [PATCH 032/213] ggated recv_thread(): Reject request with more than MAXPHYS bytes of data .. to limit the amount of memory we (try to) allocate on behalf of the client without knowing whether or not the client actually intents to use it. MAXPHYS is the hardcoded limit in ggatec so anything above it is suspicious and could be a DoS attempt. This commit forces users who like to tune MAXPHYS to make sure the value used by ggated is not below the one used by ggatec. While not ideal, this seems preferable to the DoS risk. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6b53dec..22810b0 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -671,6 +671,16 @@ recv_thread(void *arg) } /* + * Limit the amount of memory we allocate on behalf of + * the client. MAXPHYS is the hard limit in ggatec, + * values above it are thus pretty suspicious. + */ + if (req->r_length > MAXPHYS) { + g_gate_xlog("Request length above MAXPHYS: %u > %u", + (unsigned)req->r_length, MAXPHYS); + } + + /* * Allocate memory for data. */ req->r_data = malloc_waitok(req->r_length); -- 2.7.0 From 3a0c3bc3477923fd4edfe9225ed6c2a4b17e4359 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Apr 2015 14:04:31 +0200 Subject: [PATCH 033/213] ggatec: Add support for SOCKS5 with domain names --- sbin/ggate/ggatec/ggatec.c | 104 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 5 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ea4f701..80a653d 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -64,6 +64,8 @@ static unsigned flags = 0; static int force = 0; static unsigned queue_size = G_GATE_QUEUE_SIZE; static unsigned port = G_GATE_PORT; +static char *socks_dest = NULL; +static unsigned dest_port = 3080; static off_t mediasize; static unsigned sectorsize = 0; static unsigned timeout = G_GATE_TIMEOUT; @@ -78,9 +80,11 @@ usage(void) fprintf(stderr, "usage: %s create [-nv] [-o ] [-p port] " "[-q queue_size] [-R rcvbuf] [-S sndbuf] [-s sectorsize] " - "[-t timeout] [-u unit] \n", getprogname()); + "[-t timeout] [-T :] [-u unit] \n", + getprogname()); fprintf(stderr, " %s rescue [-nv] [-o ] [-p port] " - "[-R rcvbuf] [-S sndbuf] <-u unit> \n", getprogname()); + "[-R rcvbuf] [-S sndbuf] [-T :] <-u unit> " + " \n", getprogname()); fprintf(stderr, " %s destroy [-f] <-u unit>\n", getprogname()); fprintf(stderr, " %s list [-v] [-u unit]\n", getprogname()); exit(EXIT_FAILURE); @@ -238,6 +242,69 @@ recv_thread(void *arg __unused) pthread_exit(NULL); } +static void +negotiate_socks_connection(int sfd) +{ + struct negotiation_request { + char version; + char nmethods; + char method; + } neg_request; + struct socks_request { + char version; + char cmd; + char reserved; + char address_type; + char host_length; + char dest[255 + 2]; + } socks_request; + char response[10]; + size_t request_length; + size_t host_length; + + host_length = strlen(socks_dest); + + neg_request.version = '\x05'; + neg_request.nmethods = '\x01'; /* We support one method: */ + neg_request.method = '\x00'; /* no authentication */ + + g_gate_log(LOG_DEBUG, "Starting SOCKS negotiation."); + if (g_gate_send(sfd, &neg_request, sizeof(neg_request), MSG_NOSIGNAL) == -1) + g_gate_xlog("Failed to send SOCKS negotiation request."); + + if (g_gate_recv(sfd, &response, sizeof(response), MSG_WAITALL) != 2) + g_gate_xlog("Failed to read SOCKS negotiation response."); + + if (response[0] != '\x05' || response[1] != '\x00') + g_gate_xlog("SOCKS negotiation failed."); + + g_gate_log(LOG_DEBUG, "Negotiated SOCKS5. " + "Requesting connection to %s:%d.", socks_dest, dest_port); + + socks_request.version = '\x05'; + socks_request.cmd = '\x01'; /* Connect */ + socks_request.reserved = '\x00'; + socks_request.address_type = '\x03'; /* Address is domain name */; + socks_request.host_length = (char)host_length; + strncpy(socks_request.dest, socks_dest, host_length); + socks_request.dest[host_length] = (char)((dest_port >> 8) & 0xff); + socks_request.dest[host_length + 1] = (char)(dest_port & 0xff); + request_length = sizeof(socks_request) - sizeof(socks_request.dest) + + host_length + 2; + + if (g_gate_send(sfd, &socks_request, request_length, MSG_NOSIGNAL) == -1) + g_gate_xlog("Failed to send SOCKS5 request."); + + if (g_gate_recv(sfd, &response, sizeof(response), MSG_WAITALL) != sizeof(response)) + g_gate_xlog("Failed to read SOCKS5 response."); + + if (response[0] != '\x05' || response[1] != '\x00') + g_gate_xlog("Failed to SOCKS5 connect to %s:%d", + socks_dest, dest_port); + + g_gate_log(LOG_INFO, "Connected to: %s:%d.", socks_dest, dest_port); +} + static int handshake(int dir) { @@ -276,6 +343,9 @@ handshake(int dir) g_gate_log(LOG_INFO, "Connected to the server: %s:%d.", host, port); + if (socks_dest != NULL) + negotiate_socks_connection(sfd); + /* * Create and send version packet. */ @@ -455,8 +525,13 @@ g_gatec_create(void) ggioc.gctl_maxcount = queue_size; ggioc.gctl_timeout = timeout; ggioc.gctl_unit = unit; - snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), "%s:%u %s", host, - port, path); + if (socks_dest != NULL) + snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), + "socks5://%s:%u -> %s:%u %s", host, + port, socks_dest, dest_port, path); + else + snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), "%s:%u %s", + host, port, path); g_gate_ioctl(G_GATE_CMD_CREATE, &ggioc); if (unit == -1) { printf("%s%u\n", G_GATE_PROVIDER_NAME, ggioc.gctl_unit); @@ -505,8 +580,9 @@ main(int argc, char *argv[]) argv += 1; for (;;) { int ch; + char *p; - ch = getopt(argc, argv, "fno:p:q:R:S:s:t:u:v"); + ch = getopt(argc, argv, "fno:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { @@ -574,6 +650,24 @@ main(int argc, char *argv[]) if (sectorsize == 0 && errno != 0) errx(EXIT_FAILURE, "Invalid sectorsize."); break; + case 'T': + if (action != CREATE && action != RESCUE) + usage(); + socks_dest = optarg; + p = strchr(socks_dest, ':'); + if (p != NULL) { + errno = 0; + *p = '\0'; + p++; + dest_port = strtoul(p, NULL, 10); + if (dest_port == 0 && errno != 0) + errx(EXIT_FAILURE, + "Invalid socks5t port: %s.", p); + } + if (strlen(socks_dest) > (size_t)255) + errx(EXIT_FAILURE, + "Socks destination address too long."); + break; case 't': if (action != CREATE) usage(); -- 2.7.0 From 517625d6b102a8c6643daefd1b55f529df8a4ca8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 16:44:32 +0200 Subject: [PATCH 034/213] ggatec: Deduplicate information in debug output --- sbin/ggate/ggatec/ggatec.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 80a653d..ed5b949 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -176,8 +176,9 @@ send_thread(void *arg __unused) pthread_kill(recvtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Sent %zd bytes (offset=%llu, " - "size=%u).", data, hdr.gh_offset, hdr.gh_length); + g_gate_log(LOG_DEBUG, "Sent data packet " + "(offset=%llu, size=%u).", + hdr.gh_offset, hdr.gh_length); } } g_gate_log(LOG_DEBUG, "%s: Died.", __func__); @@ -224,16 +225,15 @@ recv_thread(void *arg __unused) ggio.gctl_length, MSG_WAITALL); if (reconnect) break; - g_gate_log(LOG_DEBUG, "Received data packet."); if (data != ggio.gctl_length) { g_gate_log(LOG_ERR, "Lost connection 4."); reconnect = 1; pthread_kill(sendtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Received %d bytes (offset=%ju, " - "size=%zu).", data, (uintmax_t)hdr.gh_offset, - (size_t)hdr.gh_length); + g_gate_log(LOG_DEBUG, "Received data packet " + "(offset=%ju, size=%zu).", + (uintmax_t)hdr.gh_offset, (size_t)hdr.gh_length); } g_gate_ioctl(G_GATE_CMD_DONE, &ggio); -- 2.7.0 From ad5ab027f1efe2c8ddcd5b4afbfad14fac8dbbd2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Apr 2015 15:26:42 +0200 Subject: [PATCH 035/213] ggatec.8: Document SOCKS5 support --- sbin/ggate/ggatec/ggatec.8 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.8 b/sbin/ggate/ggatec/ggatec.8 index 705d42f..c50416f 100644 --- a/sbin/ggate/ggatec/ggatec.8 +++ b/sbin/ggate/ggatec/ggatec.8 @@ -41,6 +41,7 @@ .Op Fl R Ar rcvbuf .Op Fl S Ar sndbuf .Op Fl s Ar sectorsize +.Op Fl T Ar remote_target:port .Op Fl t Ar timeout .Op Fl u Ar unit .Ar host @@ -53,6 +54,7 @@ .Op Fl p Ar port .Op Fl R Ar rcvbuf .Op Fl S Ar sndbuf +.Op Fl T Ar remote_target:port .Fl u Ar unit .Ar host .Ar path @@ -137,6 +139,9 @@ Sector size for .Nm ggate provider. If not specified, it is taken from the device, or set to 512 bytes for files. +.It Fl T Ar remote_host:port +Use SOCK5 to open connection to remote_host:port before switching +to the ggated protocol. .It Fl t Ar timeout Number of seconds to wait before an I/O request will be canceled. Default is 0, which means no timeout. @@ -167,6 +172,14 @@ server# ggated client# ggatec create -o ro server /dev/acd0 ggate0 client# mount_cd9660 /dev/ggate0 /cdrom + +.Ed +Connect to 127.0.1.1:9050, SOCKS5-negotiate a connection to +the Tor location hidden service czdqtfrgvizltdal.onion:1312 +and access a ZVOL: +.Bd -literal -offset indent +# ggatec create -T czdqtfrgvizltdal.onion:1312 -p 9050 \\ + 127.0.1.1 /dev/zvol/dpool/ggated/czdqtfrgvizltdal.eli .Ed .Sh SEE ALSO .Xr geom 4 , -- 2.7.0 From cdce3c7f6f61819a6545f78d2806e8619ea95f61 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:10:17 +0200 Subject: [PATCH 036/213] ggatec: Reject unexpected GGATE commands in recv_thread() --- sbin/ggate/ggatec/ggatec.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ed5b949..67a0f40 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -220,6 +220,11 @@ recv_thread(void *arg __unused) ggio.gctl_length = hdr.gh_length; ggio.gctl_error = hdr.gh_error; + if (ggio.gctl_cmd != GGATE_CMD_READ && + ggio.gctl_cmd != GGATE_CMD_WRITE) { + g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); + } + if (ggio.gctl_error == 0 && ggio.gctl_cmd == GGATE_CMD_READ) { data = g_gate_recv(recvfd, ggio.gctl_data, ggio.gctl_length, MSG_WAITALL); -- 2.7.0 From 21f490a67a17ecee24cecead873c7a7cfdc6f766 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:15:18 +0200 Subject: [PATCH 037/213] ggatec: Log if the remote side signals errors --- sbin/ggate/ggatec/ggatec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 67a0f40..dd75c64 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -225,6 +225,12 @@ recv_thread(void *arg __unused) g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } + if (ggio.gctl_error != 0) { + g_gate_log(LOG_ERR, + "Remote side signaled error %d: %s.", + ggio.gctl_error, strerror(ggio.gctl_error)); + } + if (ggio.gctl_error == 0 && ggio.gctl_cmd == GGATE_CMD_READ) { data = g_gate_recv(recvfd, ggio.gctl_data, ggio.gctl_length, MSG_WAITALL); -- 2.7.0 From 9fff5cd431253e6c9d228ad2eb16ca8fbf86e02f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:39:25 +0200 Subject: [PATCH 038/213] ggatec: Reject unsupported BIO commands Due to missing sanity checks in ggatec, requests like BIO_FLUSH and BIO_DELETE were previously accepted and sent as zero-size reads or writes. Also due to missing sanity checks, ggated did not mind. --- sbin/ggate/ggatec/ggatec.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index dd75c64..650e4f5 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,13 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + default: + g_gate_log(LOG_ERR, + "Rejecting unsupported BIO command: %d", + ggio.gctl_cmd); + ggio.gctl_error = EOPNOTSUPP; + g_gate_ioctl(G_GATE_CMD_DONE, &ggio); + continue; } hdr.gh_seq = ggio.gctl_seq; hdr.gh_offset = ggio.gctl_offset; -- 2.7.0 From 9057ea476f61a8c7830f2c5a1930fd373d2797c4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:53:30 +0200 Subject: [PATCH 039/213] ggate[cd]: Add BIO_FLUSH support Let ggated transform BIO_FLUSH requests into fsync() calls. --- sbin/ggate/ggatec/ggatec.c | 8 +++++++- sbin/ggate/ggated/ggated.c | 23 ++++++++++++++++++----- sbin/ggate/shared/ggate.h | 1 + 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 650e4f5..f91c172 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,11 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + case BIO_FLUSH: + g_gate_log(LOG_DEBUG, "FLUSH request"); + hdr.gh_cmd = GGATE_CMD_FLUSH; + assert(ggio.gctl_length == 0); + break; default: g_gate_log(LOG_ERR, "Rejecting unsupported BIO command: %d", @@ -228,7 +233,8 @@ recv_thread(void *arg __unused) ggio.gctl_error = hdr.gh_error; if (ggio.gctl_cmd != GGATE_CMD_READ && - ggio.gctl_cmd != GGATE_CMD_WRITE) { + ggio.gctl_cmd != GGATE_CMD_WRITE && + ggio.gctl_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 22810b0..2adbb68 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -657,8 +657,10 @@ recv_thread(void *arg) * Reject requests that violate assertions in disk_thread(). */ if (req->r_cmd != GGATE_CMD_READ && - req->r_cmd != GGATE_CMD_WRITE) { - g_gate_xlog("Request contains invalid command."); + req->r_cmd != GGATE_CMD_WRITE && + req->r_cmd != GGATE_CMD_FLUSH) { + g_gate_xlog("Request contains invalid command: %d", + req->r_cmd); } if (req->r_offset + req->r_length > (uintmax_t)conn->c_mediasize) { @@ -681,9 +683,10 @@ recv_thread(void *arg) } /* - * Allocate memory for data. + * Allocate memory for data, except when flushing. */ - req->r_data = malloc_waitok(req->r_length); + req->r_data = req->r_cmd != GGATE_CMD_FLUSH ? + malloc_waitok(req->r_length) : NULL; /* * Receive data to write for WRITE request. @@ -743,7 +746,9 @@ disk_thread(void *arg) /* * Check the request. */ - assert(req->r_cmd == GGATE_CMD_READ || req->r_cmd == GGATE_CMD_WRITE); + assert(req->r_cmd == GGATE_CMD_READ || + req->r_cmd == GGATE_CMD_WRITE || + req->r_cmd == GGATE_CMD_FLUSH); assert(req->r_offset + req->r_length <= (uintmax_t)conn->c_mediasize); assert((req->r_offset % conn->c_sectorsize) == 0); assert((req->r_length % conn->c_sectorsize) == 0); @@ -767,6 +772,14 @@ disk_thread(void *arg) free(req->r_data); req->r_data = NULL; break; + case GGATE_CMD_FLUSH: + g_gate_log(LOG_DEBUG, "Flushing"); + if (fsync(fd)) { + req->r_error = errno; + g_gate_log(LOG_ERR, "Flushing failed: %s", + strerror(errno)); + } + break; } if (data != (ssize_t)req->r_length) { /* Report short reads/writes as I/O errors. */ diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 898efea..5298566 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -55,6 +55,7 @@ #define GGATE_CMD_READ 0 #define GGATE_CMD_WRITE 1 +#define GGATE_CMD_FLUSH 2 extern int g_gate_devfd; extern int g_gate_verbose; -- 2.7.0 From 916a357233b81b0afbad31573f787b67bbe721b9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 29 Apr 2015 12:44:56 +0200 Subject: [PATCH 040/213] ggatec: Log the command type for hdr packets (when debugging) ... and provide more details about failed requests. --- sbin/ggate/ggatec/ggatec.c | 11 ++++++++--- sbin/ggate/shared/ggate.c | 16 ++++++++++++++++ sbin/ggate/shared/ggate.h | 1 + 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index f91c172..4d0b12f 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -166,7 +166,9 @@ send_thread(void *arg __unused) g_gate_swap2n_hdr(&hdr); data = g_gate_send(sendfd, &hdr, sizeof(hdr), MSG_NOSIGNAL); - g_gate_log(LOG_DEBUG, "Sent hdr packet."); + g_gate_log(LOG_DEBUG, "Sent hdr packet (%s).", + g_gate_cmd2str(hdr.gh_cmd)); + g_gate_swap2h_hdr(&hdr); if (reconnect) break; @@ -224,7 +226,8 @@ recv_thread(void *arg __unused) pthread_kill(sendtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Received hdr packet."); + g_gate_log(LOG_DEBUG, "Received hdr packet (%s).", + g_gate_cmd2str(hdr.gh_cmd)); ggio.gctl_seq = hdr.gh_seq; ggio.gctl_cmd = hdr.gh_cmd; @@ -240,7 +243,9 @@ recv_thread(void *arg __unused) if (ggio.gctl_error != 0) { g_gate_log(LOG_ERR, - "Remote side signaled error %d: %s.", + "%s for %d bytes at offset %d failed. " + "Error %d: %s.", g_gate_cmd2str(ggio.gctl_cmd), + ggio.gctl_length, ggio.gctl_offset, ggio.gctl_error, strerror(ggio.gctl_error)); } diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index cf9b9ca..05fc25a 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -407,3 +407,19 @@ g_gate_str2ip(const char *str) return (INADDR_NONE); return (((struct in_addr *)(void *)hp->h_addr)->s_addr); } + +const char * +g_gate_cmd2str(int cmd) +{ + + switch (cmd) { + case GGATE_CMD_READ: + return ("GGATE_CMD_READ"); + case GGATE_CMD_WRITE: + return ("GGATE_CMD_WRITE"); + case GGATE_CMD_FLUSH: + return ("GGATE_CMD_FLUSH"); + } + + return ("unknown (invalid?) GGATE command"); +} diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 5298566..3656519 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -112,6 +112,7 @@ void g_gate_socket_settings(int sfd); void g_gate_list(int unit, int verbose); #endif in_addr_t g_gate_str2ip(const char *str); +const char *g_gate_cmd2str(int cmd); /* * g_gate_swap2h_* - functions swap bytes to host byte order (from big endian). -- 2.7.0 From 5d1358a1f65b2e9bb4f21d7234aa37ffee2017a4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 6 May 2015 15:55:08 +0200 Subject: [PATCH 041/213] ggated disk_thread(): Include the command in the debug output --- sbin/ggate/ggated/ggated.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 2adbb68..fa11d8b 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -753,7 +753,8 @@ disk_thread(void *arg) assert((req->r_offset % conn->c_sectorsize) == 0); assert((req->r_length % conn->c_sectorsize) == 0); - g_gate_log(LOG_DEBUG, "%s: offset=%jd length=%u", __func__, + g_gate_log(LOG_DEBUG, "%s: cmd=%s offset=%jd length=%u", + __func__, g_gate_cmd2str(req->r_cmd), (intmax_t)req->r_offset, (unsigned)req->r_length); /* -- 2.7.0 From 9885d3b657f4dc7cdd6128db83a30d03082a2856 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 5 May 2015 17:39:16 +0200 Subject: [PATCH 042/213] ggate[cd]: Add BIO_DELETE support On the ggated side the requests are translated into writes of zero which ZFS will convert into BIO_DELETE requests again when zle compression is enabled. --- sbin/ggate/ggatec/ggatec.c | 5 +++++ sbin/ggate/ggated/ggated.c | 47 +++++++++++++++++++++++++++++++++++++++------- sbin/ggate/shared/ggate.c | 2 ++ sbin/ggate/shared/ggate.h | 1 + 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 4d0b12f..e70cf55 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,10 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + case BIO_DELETE: + g_gate_log(LOG_DEBUG, "DELETE request"); + hdr.gh_cmd = GGATE_CMD_DELETE; + break; case BIO_FLUSH: g_gate_log(LOG_DEBUG, "FLUSH request"); hdr.gh_cmd = GGATE_CMD_FLUSH; @@ -237,6 +241,7 @@ recv_thread(void *arg __unused) if (ggio.gctl_cmd != GGATE_CMD_READ && ggio.gctl_cmd != GGATE_CMD_WRITE && + ggio.gctl_cmd != GGATE_CMD_DELETE && ggio.gctl_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index fa11d8b..c640b57 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -636,6 +636,7 @@ recv_thread(void *arg) * Get header packet. */ req = malloc_waitok(sizeof(*req)); + memset(req, 0, sizeof(*req)); data = g_gate_recv(fd, &req->r_hdr, sizeof(req->r_hdr), MSG_WAITALL); if (data == 0) { @@ -658,6 +659,7 @@ recv_thread(void *arg) */ if (req->r_cmd != GGATE_CMD_READ && req->r_cmd != GGATE_CMD_WRITE && + req->r_cmd != GGATE_CMD_DELETE && req->r_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Request contains invalid command: %d", req->r_cmd); @@ -677,21 +679,16 @@ recv_thread(void *arg) * the client. MAXPHYS is the hard limit in ggatec, * values above it are thus pretty suspicious. */ - if (req->r_length > MAXPHYS) { + if (req->r_length > MAXPHYS && req->r_cmd != GGATE_CMD_DELETE) { g_gate_xlog("Request length above MAXPHYS: %u > %u", (unsigned)req->r_length, MAXPHYS); } /* - * Allocate memory for data, except when flushing. - */ - req->r_data = req->r_cmd != GGATE_CMD_FLUSH ? - malloc_waitok(req->r_length) : NULL; - - /* * Receive data to write for WRITE request. */ if (req->r_cmd == GGATE_CMD_WRITE) { + req->r_data = malloc_waitok(req->r_length); g_gate_log(LOG_DEBUG, "Waiting for %u bytes of data...", req->r_length); data = g_gate_recv(fd, req->r_data, req->r_length, @@ -718,6 +715,34 @@ recv_thread(void *arg) } } +static ssize_t +delete_range(int fd, size_t length, off_t offset) +{ + static char zeros[MAXPHYS]; + size_t written; + + written = 0; + + do + { + int ret; + size_t bytes_left; + size_t chunk_size; + + bytes_left = length - written; + chunk_size = bytes_left > MAXPHYS ? MAXPHYS : bytes_left; + ret = pwrite(fd, zeros, chunk_size, offset + written); + if (ret == -1) + return (written); + written += ret; + } while (written < length); + + g_gate_log(LOG_DEBUG, "Overwritten %u bytes at offset %jd with zeros", + written, (intmax_t)offset); + + return (written); +} + static void * disk_thread(void *arg) { @@ -748,6 +773,7 @@ disk_thread(void *arg) */ assert(req->r_cmd == GGATE_CMD_READ || req->r_cmd == GGATE_CMD_WRITE || + req->r_cmd == GGATE_CMD_DELETE || req->r_cmd == GGATE_CMD_FLUSH); assert(req->r_offset + req->r_length <= (uintmax_t)conn->c_mediasize); assert((req->r_offset % conn->c_sectorsize) == 0); @@ -763,9 +789,16 @@ disk_thread(void *arg) data = 0; switch (req->r_cmd) { case GGATE_CMD_READ: + assert(req->r_data == NULL); + req->r_data = malloc_waitok(req->r_length); data = pread(fd, req->r_data, req->r_length, req->r_offset); break; + case GGATE_CMD_DELETE: + data = delete_range(fd, req->r_length, + req->r_offset); + assert((size_t)data <= req->r_length); + break; case GGATE_CMD_WRITE: data = pwrite(fd, req->r_data, req->r_length, req->r_offset); diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index 05fc25a..7fe37cc 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -417,6 +417,8 @@ g_gate_cmd2str(int cmd) return ("GGATE_CMD_READ"); case GGATE_CMD_WRITE: return ("GGATE_CMD_WRITE"); + case GGATE_CMD_DELETE: + return ("GGATE_CMD_DELETE"); case GGATE_CMD_FLUSH: return ("GGATE_CMD_FLUSH"); } diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 3656519..e5792e3 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -56,6 +56,7 @@ #define GGATE_CMD_READ 0 #define GGATE_CMD_WRITE 1 #define GGATE_CMD_FLUSH 2 +#define GGATE_CMD_DELETE 3 extern int g_gate_devfd; extern int g_gate_verbose; -- 2.7.0 From 7b286faab129e45dcd01ab54782af556f04684cb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 29 Apr 2015 10:55:40 +0200 Subject: [PATCH 043/213] ggated send_thread(): Assert that we only send data for read requests --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index c640b57..8e564e1 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -883,6 +883,7 @@ send_thread(void *arg) g_gate_log(LOG_DEBUG, "Sent hdr packet."); g_gate_swap2h_hdr(&req->r_hdr); if (req->r_data != NULL) { + assert(req->r_cmd == GGATE_CMD_READ); data = g_gate_send(fd, req->r_data, req->r_length, 0); if (data != (ssize_t)req->r_length) { g_gate_xlog("Error while sending data: %s.", -- 2.7.0 From fe4482c4d9a8468a22a136f3f11d7c3a7cec17e9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:00:04 +0200 Subject: [PATCH 044/213] ggated: Open the listening socket CLOEXEC --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 8e564e1..49e9c58 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1109,7 +1109,7 @@ main(int argc, char *argv[]) signal(SIGCHLD, SIG_IGN); signal(SIGPIPE, SIG_IGN); - sfd = socket(AF_INET, SOCK_STREAM, 0); + sfd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (sfd == -1) g_gate_xlog("Cannot open stream socket: %s.", strerror(errno)); bzero(&serv, sizeof(serv)); -- 2.7.0 From 8169e0f2b03e52954b54cdf78ea6da6e5dc05501 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:31:46 +0200 Subject: [PATCH 045/213] ggated: Fix another socket leak --- sbin/ggate/ggated/ggated.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 49e9c58..383386b 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -341,6 +341,11 @@ exports_check(struct ggd_export *ex, struct g_gate_cinit *cinit, return (EPERM); } } + if (conn->c_diskfd != -1) { + g_gate_log(LOG_DEBUG, "Requested file %s is already open: %d", + ex->e_path, conn->c_diskfd); + return(0); + } if ((conn->c_flags & GGATE_FLAG_RDONLY) != 0) flags = O_RDONLY; else if ((conn->c_flags & GGATE_FLAG_WRONLY) != 0) -- 2.7.0 From 1e52aee35df5a00a58af58bec0c10f130eeb840c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 30 Apr 2015 11:52:06 +0200 Subject: [PATCH 046/213] ggated recv_thread(): In case of read-only files, only accept read commands Accepting write commands etc. is not a security problem because the file descriptor isn't writeable anyway, but accepting requests other than reads could hide client bugs. --- sbin/ggate/ggated/ggated.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 383386b..5ae9d2f 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -679,6 +679,12 @@ recv_thread(void *arg) "not fit sector size."); } + if ((conn->c_flags & GGATE_FLAG_RDONLY) != 0 + && req->r_cmd != GGATE_CMD_READ) { + g_gate_xlog("%s request received for read-only file", + g_gate_cmd2str(req->r_cmd)); + } + /* * Limit the amount of memory we allocate on behalf of * the client. MAXPHYS is the hard limit in ggatec, -- 2.7.0 From 30cb57c9429674970cf62db1be687c7e382129f7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 3 May 2015 14:02:02 +0200 Subject: [PATCH 047/213] ggatec: Add log-to-file support --- sbin/ggate/ggatec/ggatec.c | 5 ++++- sbin/ggate/shared/ggate.c | 26 +++++++++++++++++++++----- sbin/ggate/shared/ggate.h | 1 + 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index e70cf55..ba68977 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -616,7 +616,7 @@ main(int argc, char *argv[]) int ch; char *p; - ch = getopt(argc, argv, "fno:p:q:R:S:s:t:T:u:v"); + ch = getopt(argc, argv, "fl:no:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { @@ -625,6 +625,9 @@ main(int argc, char *argv[]) usage(); force = 1; break; + case 'l': + g_gate_open_log(optarg); + break; case 'n': if (action != CREATE && action != RESCUE) usage(); diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index 7fe37cc..b5799d6 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -26,6 +26,7 @@ * $FreeBSD$ */ +#define _WITH_DPRINTF #include #include #include @@ -59,13 +60,23 @@ int g_gate_devfd = -1; int g_gate_verbose = 0; +static int g_gate_logfd = -1; +void +g_gate_open_log(const char *logfile) +{ + + g_gate_logfd = open(logfile, O_CREAT | O_WRONLY | O_APPEND, S_IWUSR |S_IRUSR); + if (g_gate_logfd == -1) { + g_gate_xlog("Failed to open %s: %s", logfile, strerror(errno)); + } +} void g_gate_vlog(int priority, const char *message, va_list ap) { - if (g_gate_verbose) { + if (g_gate_verbose || g_gate_logfd != -1) { const char *prefix; switch (priority) { @@ -87,10 +98,15 @@ g_gate_vlog(int priority, const char *message, va_list ap) default: prefix = "unknown"; } - - printf("%s: ", prefix); - vprintf(message, ap); - printf("\n"); + if (g_gate_logfd == -1) { + printf("%s: ", prefix); + vprintf(message, ap); + printf("\n"); + } else if (g_gate_verbose || priority != LOG_DEBUG) { + dprintf(g_gate_logfd, "%s: ", prefix); + vdprintf(g_gate_logfd, message, ap); + dprintf(g_gate_logfd, "\n"); + } } else { if (priority != LOG_DEBUG) vsyslog(priority, message, ap); diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index e5792e3..51f425d 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -95,6 +95,7 @@ struct g_gate_hdr { uint16_t gh_error; /* error value (0 if ok) */ } __packed; +void g_gate_open_log(const char *logfile); void g_gate_vlog(int priority, const char *message, va_list ap); void g_gate_log(int priority, const char *message, ...); void g_gate_xvlog(const char *message, va_list ap) __dead2; -- 2.7.0 From 28f7784c380ab1884d064de140965ecf506f236e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 30 Apr 2015 13:52:39 +0200 Subject: [PATCH 048/213] ggate[cd]: Add Jail and Capsicum support The capsicum support for ggatec is incomplete and only enabled if the -c flag is used as it currently prevents ggatec from reconnecting which is very inconvenient. --- sbin/ggate/ggatec/ggatec.8 | 8 +++ sbin/ggate/ggatec/ggatec.c | 37 +++++++++--- sbin/ggate/ggated/ggated.c | 5 ++ sbin/ggate/shared/ggate.c | 137 +++++++++++++++++++++++++++++++++++++++++++++ sbin/ggate/shared/ggate.h | 2 + 5 files changed, 182 insertions(+), 7 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.8 b/sbin/ggate/ggatec/ggatec.8 index c50416f..f348dce 100644 --- a/sbin/ggate/ggatec/ggatec.8 +++ b/sbin/ggate/ggatec/ggatec.8 @@ -33,6 +33,7 @@ .Sh SYNOPSIS .Nm .Cm create +.Op Fl c .Op Fl n .Op Fl v .Op Fl o Cm ro | wo | rw @@ -48,6 +49,7 @@ .Ar path .Nm .Cm rescue +.Op Fl c .Op Fl n .Op Fl v .Op Fl o Cm ro | wo | rw @@ -104,6 +106,12 @@ providers. .Pp Available options: .Bl -tag -width ".Fl s Cm ro | wo | rw" +.It Fl c +Enter capsicum sandbox. +Currently this prevents +.Nm ggatec +from reconnection which is somewhat inconvenient. +The flag will go away once this is fixed. .It Fl f Forcibly destroy .Nm ggate diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ba68977..ad2db88 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include "ggate.h" @@ -59,6 +60,8 @@ static enum { UNSET, CREATE, DESTROY, LIST, RESCUE } action = UNSET; static const char *path = NULL; static const char *host = NULL; +static in_addr_t host_ip; +static const char *logfile = NULL; static int unit = G_GATE_UNIT_AUTO; static unsigned flags = 0; static int force = 0; @@ -73,6 +76,7 @@ static int sendfd, recvfd; static uint32_t token; static pthread_t sendtd, recvtd; static int reconnect; +static int drop_capabilities = 0; static void usage(void) @@ -353,7 +357,7 @@ handshake(int dir) */ bzero(&serv, sizeof(serv)); serv.sin_family = AF_INET; - serv.sin_addr.s_addr = g_gate_str2ip(host); + serv.sin_addr.s_addr = host_ip; if (serv.sin_addr.s_addr == INADDR_NONE) { g_gate_log(LOG_DEBUG, "Invalid IP/host name: %s.", host); return (-1); @@ -461,7 +465,7 @@ static void mydaemon(void) { - if (g_gate_verbose > 0) + if (logfile == NULL && g_gate_verbose > 0) return; if (daemon(0, 0) == 0) return; @@ -526,6 +530,10 @@ g_gatec_loop(void) signal(SIGUSR1, signop); for (;;) { g_gatec_start(); + + if (cap_sandboxed()) + g_gate_xlog("Got disconnected while being sandboxed."); + g_gate_log(LOG_NOTICE, "Disconnected [%s %s]. Connecting...", host, path); while (!g_gatec_connect()) { @@ -548,9 +556,6 @@ g_gatec_create(void) if (!g_gatec_connect()) g_gate_xlog("Cannot connect: %s.", strerror(errno)); - /* - * Ok, got both sockets, time to create provider. - */ memset(&ggioc, 0, sizeof(ggioc)); ggioc.gctl_version = G_GATE_VERSION; ggioc.gctl_mediasize = mediasize; @@ -573,6 +578,9 @@ g_gatec_create(void) } unit = ggioc.gctl_unit; + if (drop_capabilities) + g_gate_drop_capabilities(sendfd, recvfd); + mydaemon(); g_gatec_loop(); } @@ -585,6 +593,9 @@ g_gatec_rescue(void) if (!g_gatec_connect()) g_gate_xlog("Cannot connect: %s.", strerror(errno)); + if (drop_capabilities) + g_gate_drop_capabilities(sendfd, recvfd); + ggioc.gctl_version = G_GATE_VERSION; ggioc.gctl_unit = unit; ggioc.gctl_seq = 0; @@ -616,17 +627,21 @@ main(int argc, char *argv[]) int ch; char *p; - ch = getopt(argc, argv, "fl:no:p:q:R:S:s:t:T:u:v"); + ch = getopt(argc, argv, "cfl:no:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { + case 'c': + drop_capabilities = 1; + force = 1; + break; case 'f': if (action != DESTROY) usage(); force = 1; break; case 'l': - g_gate_open_log(optarg); + logfile = optarg; break; case 'n': if (action != CREATE && action != RESCUE) @@ -738,7 +753,11 @@ main(int argc, char *argv[]) g_gate_load_module(); g_gate_open_device(); host = argv[0]; + host_ip = g_gate_str2ip(host); path = argv[1]; + if (logfile != NULL) + g_gate_open_log(logfile); + g_gate_drop_privs("hast", host_ip); g_gatec_create(); break; case DESTROY: @@ -762,7 +781,11 @@ main(int argc, char *argv[]) } g_gate_open_device(); host = argv[0]; + host_ip = g_gate_str2ip(host); path = argv[1]; + if (logfile != NULL) + g_gate_open_log(logfile); + g_gate_drop_privs("hast", host_ip); g_gatec_rescue(); break; case UNSET: diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 5ae9d2f..e7dbfbf 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -545,6 +545,11 @@ connection_launch(struct ggd_connection *conn) } g_gate_log(LOG_DEBUG, "Process created [%s].", conn->c_path); + if (getuid() == 0) + g_gate_drop_privs("hast", bindaddr); + + g_gate_drop_capabilities(conn->c_sendfd, conn->c_recvfd); + /* * Create condition variables and mutexes for in-queue and out-queue * synchronization. diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index b5799d6..13277ea 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -53,6 +53,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include "ggate.h" @@ -441,3 +446,135 @@ g_gate_cmd2str(int cmd) return ("unknown (invalid?) GGATE command"); } + +/* + * The functions below are based on drop_privs() from ../../hastd/subr.c + * + * Changes: + * - HAST_USER replaced with ggate_user option + * - pjdlog_* replaced with g_gate_xlog(). + * - Don't fall back to chroot if jailing fails. + */ +#define PJDLOG_VERIFY assert +void +g_gate_drop_privs(const char *ggate_user, in_addr_t jail_address) +{ + char jailhost[32]; + struct jail jailst; + struct passwd *pw; + uid_t ruid, euid, suid; + gid_t rgid, egid, sgid; + gid_t gidset[1]; + struct in_addr jail_ip; + /* + * According to getpwnam(3) we have to clear errno before calling the + * function to be able to distinguish between an error and missing + * entry (with is not treated as error by getpwnam(3)). + */ + errno = 0; + pw = getpwnam(ggate_user); + if (pw == NULL) { + if (errno != 0) { + g_gate_xlog("Unable to find info about '%s' user", + ggate_user); + } else { + g_gate_xlog("'%s' user doesn't exist.", ggate_user); + } + } + + jail_ip.s_addr = jail_address; + + bzero(&jailst, sizeof(jailst)); + jailst.version = JAIL_API_VERSION; + jailst.path = pw->pw_dir; + (void)snprintf(jailhost, sizeof(jailhost), "%s-jail", getprogname()); + jailst.hostname = jailhost; + jailst.jailname = NULL; + jailst.ip4s = 1; + jailst.ip4 = &jail_ip; + jailst.ip6s = 0; + jailst.ip6 = NULL; + if (jail(&jailst) == -1) { + g_gate_xlog("Unable to jail process in directory %s", pw->pw_dir); + } + PJDLOG_VERIFY(chdir("/") == 0); + gidset[0] = pw->pw_gid; + if (setgroups(1, gidset) == -1) { + g_gate_xlog("Unable to set groups to gid %u", + (unsigned int)pw->pw_gid); + } + if (setgid(pw->pw_gid) == -1) { + g_gate_xlog("Unable to set gid to %u", + (unsigned int)pw->pw_gid); + } + if (setuid(pw->pw_uid) == -1) { + g_gate_xlog("Unable to set uid to %u", + (unsigned int)pw->pw_uid); + } + + /* + * Better be sure that everything succeeded. + */ + PJDLOG_VERIFY(getresuid(&ruid, &euid, &suid) == 0); + PJDLOG_VERIFY(ruid == pw->pw_uid); + PJDLOG_VERIFY(euid == pw->pw_uid); + PJDLOG_VERIFY(suid == pw->pw_uid); + PJDLOG_VERIFY(getresgid(&rgid, &egid, &sgid) == 0); + PJDLOG_VERIFY(rgid == pw->pw_gid); + PJDLOG_VERIFY(egid == pw->pw_gid); + PJDLOG_VERIFY(sgid == pw->pw_gid); + PJDLOG_VERIFY(getgroups(0, NULL) == 1); + PJDLOG_VERIFY(getgroups(1, gidset) == 1); + PJDLOG_VERIFY(gidset[0] == pw->pw_gid); + + g_gate_log(LOG_DEBUG, "Privileges successfully dropped using " + "jail+setgid+setuid."); +} + +int +g_gate_drop_capabilities(int sendfd, int recvfd) +{ + cap_rights_t rights; + static const unsigned long ggatecmds[] = { + G_GATE_CMD_START, + G_GATE_CMD_DONE, + G_GATE_CMD_CANCEL, + }; + + if (cap_enter() != 0) { + g_gate_xlog("Failed to sandbox using capsicum"); + } + + cap_rights_init(&rights, CAP_PREAD, CAP_PWRITE); + if (cap_rights_limit(sendfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on sendfd %d", sendfd); + } + if (cap_rights_limit(recvfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on recvfd %d", recvfd); + } + + /* Only the client uses this. */ + if (g_gate_devfd != -1) { + cap_rights_init(&rights, CAP_IOCTL, CAP_PREAD, CAP_PWRITE); + if (cap_rights_limit(g_gate_devfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability rights " + "to CAP_IOCTL on ggate descriptor"); + } + if (cap_ioctls_limit(g_gate_devfd, ggatecmds, + sizeof(ggatecmds) / sizeof(ggatecmds[0])) == -1) { + g_gate_xlog("Unable to limit allowed ggate ioctls"); + } + } + cap_rights_init(&rights, CAP_PWRITE); + if (g_gate_logfd != -1 && + cap_rights_limit(g_gate_logfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on logfd %d", g_gate_logfd); + } + + g_gate_log(LOG_DEBUG, "Entered Capsicum sandbox"); + + return (0); +} diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 51f425d..cdc3cbf 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -115,6 +115,8 @@ void g_gate_list(int unit, int verbose); #endif in_addr_t g_gate_str2ip(const char *str); const char *g_gate_cmd2str(int cmd); +void g_gate_drop_privs(const char *ggate_user, in_addr_t jail_address); +int g_gate_drop_capabilities(int sendfd, int recvfd); /* * g_gate_swap2h_* - functions swap bytes to host byte order (from big endian). -- 2.7.0 From abe2c1a4ebef0794d919770c3567dc8216742ce5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 28 Apr 2015 13:02:25 +0200 Subject: [PATCH 049/213] Bump GGATE_VERSION due to FLUSH and DELETE support and various bug fixes Unpatched ggate[cd] versions may cause data corruption so we no longer want to speak to them. --- sbin/ggate/shared/ggate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index cdc3cbf..e764665 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -40,7 +40,7 @@ #define G_GATE_TIMEOUT 0 #define GGATE_MAGIC "GEOM_GATE " -#define GGATE_VERSION 0 +#define GGATE_VERSION 1 #define GGATE_FLAG_RDONLY 0x0001 #define GGATE_FLAG_WRONLY 0x0002 -- 2.7.0 From 1d568fd378f8b6f46853d58c6aace52918f08e9a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 9 Aug 2015 15:20:48 +0200 Subject: [PATCH 050/213] Use dedicated users for ggatec and ggated --- etc/group | 2 ++ etc/master.passwd | 2 ++ sbin/ggate/ggatec/ggatec.c | 2 +- sbin/ggate/ggated/ggated.c | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/group b/etc/group index c20f1d9..3136230 100644 --- a/etc/group +++ b/etc/group @@ -31,5 +31,7 @@ audit:*:77: www:*:80: _ypldap:*:160: hast:*:845: +ggatec:*:846: +ggated:*:847: nogroup:*:65533: nobody:*:65534: diff --git a/etc/master.passwd b/etc/master.passwd index e603192..430fd2c 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -24,4 +24,6 @@ auditdistd:*:78:77::0:0:Auditdistd unprivileged user:/var/empty:/usr/sbin/nologi www:*:80:80::0:0:World Wide Web Owner:/nonexistent:/usr/sbin/nologin _ypldap:*:160:160::0:0:YP Ldap unprivileged user:/var/empty:/usr/sbin/nologin hast:*:845:845::0:0:HAST unprivileged user:/var/empty:/usr/sbin/nologin +ggatec:*:846:846::0:0:ggatec unprivileged user:/var/empty:/usr/sbin/nologin +ggated:*:847:847::0:0:ggated unprivileged user:/var/empty:/usr/sbin/nologin nobody:*:65534:65534::0:0:Unprivileged user:/nonexistent:/usr/sbin/nologin diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ad2db88..cd33e12 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -757,7 +757,7 @@ main(int argc, char *argv[]) path = argv[1]; if (logfile != NULL) g_gate_open_log(logfile); - g_gate_drop_privs("hast", host_ip); + g_gate_drop_privs("ggatec", host_ip); g_gatec_create(); break; case DESTROY: diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index e7dbfbf..4397253 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -546,7 +546,7 @@ connection_launch(struct ggd_connection *conn) g_gate_log(LOG_DEBUG, "Process created [%s].", conn->c_path); if (getuid() == 0) - g_gate_drop_privs("hast", bindaddr); + g_gate_drop_privs("ggated", bindaddr); g_gate_drop_capabilities(conn->c_sendfd, conn->c_recvfd); -- 2.7.0 From 6dd163c5eecf15c9cdfe8bad59b5763febf6d2ba Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Jan 2015 18:50:18 +0100 Subject: [PATCH 051/213] bge(4): Default to disallowing ASF It causes watchdog timeouts and undiagnosed permanent unresponsivenes on at least the 'CHIP ID 0x05784100; ASIC REV 0x5784; CHIP REV 0x57841;' in the evo-iv08 DL120 G6 I'm using for testing. I don't have time to debug the underlying cause right now and users who actually want ASF and have systems where it works can always enable it through loader.conf. --- share/man/man4/bge.4 | 4 ++-- sys/dev/bge/if_bge.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/share/man/man4/bge.4 b/share/man/man4/bge.4 index 474b48f..060d22d 100644 --- a/share/man/man4/bge.4 +++ b/share/man/man4/bge.4 @@ -31,7 +31,7 @@ .\" .\" $FreeBSD$ .\" -.Dd January 19, 2012 +.Dd January 19, 2015 .Dt BGE 4 .Os .Sh NAME @@ -196,7 +196,7 @@ prompt before booting the kernel, or stored in .It Va hw.bge.allow_asf Allow the ASF feature for cooperating with IPMI. Can cause system lockup problems on a small number of systems. -Enabled by default. +Disabled by default. .It Va dev.bge.%d.msi Non-zero value enables MSI support on the Ethernet hardware. The default value is 1. diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c index 007ec63..91c30ed 100644 --- a/sys/dev/bge/if_bge.c +++ b/sys/dev/bge/if_bge.c @@ -541,7 +541,7 @@ static devclass_t bge_devclass; DRIVER_MODULE(bge, pci, bge_driver, bge_devclass, 0, 0); DRIVER_MODULE(miibus, bge, miibus_driver, miibus_devclass, 0, 0); -static int bge_allow_asf = 1; +static int bge_allow_asf = 0; static SYSCTL_NODE(_hw, OID_AUTO, bge, CTLFLAG_RD, 0, "BGE driver parameters"); SYSCTL_INT(_hw_bge, OID_AUTO, allow_asf, CTLFLAG_RDTUN, &bge_allow_asf, 0, -- 2.7.0 From a3e83b0d7b8f67cb19350f7fefb20a372ce54340 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Jan 2015 21:25:47 +0100 Subject: [PATCH 052/213] Assign random IP id values by default so users don't have to clown around with the sysctl themselves --- sys/netinet/ip_id.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/netinet/ip_id.c b/sys/netinet/ip_id.c index b0efbee..9d4c63d 100644 --- a/sys/netinet/ip_id.c +++ b/sys/netinet/ip_id.c @@ -137,8 +137,8 @@ static void ipid_sysuninit(void); SYSCTL_DECL(_net_inet_ip); SYSCTL_PROC(_net_inet_ip, OID_AUTO, random_id, CTLTYPE_INT | CTLFLAG_VNET | CTLFLAG_RW, - &VNET_NAME(ip_do_randomid), 0, sysctl_ip_randomid, "IU", - "Assign random ip_id values"); + &VNET_NAME(ip_do_randomid), 1, sysctl_ip_randomid, "IU", + "Assign random ip_id values. Important for Tor relays and a good idea in general."); SYSCTL_INT(_net_inet_ip, OID_AUTO, rfc6864, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip_rfc6864), 0, "Use constant IP ID for atomic datagrams"); -- 2.7.0 From 6aa525e8e8f71ab894508c5e6f16168e152357d0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 11 Feb 2015 12:19:24 +0100 Subject: [PATCH 053/213] Add vendor copyright ... after putting on my robe and "police educational technican" hat. This commit is optional. Feel free to import any other ElectroBSD commit without including this one. --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index e3c1e40..12d421e 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -30,7 +30,7 @@ /* Add a FreeBSD vendor copyright here */ #define COPYRIGHT_Vendor \ - "" + "Copyright (c) 2010-2015 Fabian Keil - IT-Beratung und Polizei-Erziehung\n" /* FreeBSD */ #define COPYRIGHT_FreeBSD \ -- 2.7.0 From a5345b6c368f258a62d2f3f9117e3da3b3a4bf64 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 13 Feb 2015 19:37:33 +0100 Subject: [PATCH 054/213] OpenZFS: fix a comment typo --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c index 6a20d25..09e0d33 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c @@ -1448,7 +1448,7 @@ int zfs_frag_table[FRAGMENTATION_TABLE_SIZE] = { }; /* - * Calclate the metaslab's fragmentation metric. A return value + * Calculate the metaslab's fragmentation metric. A return value * of ZFS_FRAG_INVALID means that the metaslab has not been upgraded and does * not support this metric. Otherwise, the return value should be in the * range [0, 100]. -- 2.7.0 From a9bc6675eeee3f375395eda340cf706ea1777297 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 13 Feb 2015 17:56:07 +0100 Subject: [PATCH 055/213] range_tree_destroy(): Optionally tolerate non-zero rt->rt_space --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c index 6422fd1..842480a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c @@ -150,7 +150,11 @@ range_tree_create(range_tree_ops_t *ops, void *arg, kmutex_t *lp) void range_tree_destroy(range_tree_t *rt) { - VERIFY0(rt->rt_space); + if (rt->rt_space != 0) { + zfs_panic_recover("zfs: range_tree_destroy(): " + "rt->rt_space != 0: %llx", rt->rt_space); + return; + } if (rt->rt_ops != NULL) rt->rt_ops->rtop_destroy(rt, rt->rt_arg); -- 2.7.0 From df8b165de15ddfe7f126a2eb92662ebc4e635e4e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 Feb 2015 11:38:09 +0100 Subject: [PATCH 056/213] newvers.sh: Set TYPE to ElectroBSD --- sys/conf/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 sys/conf/newvers.sh diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh old mode 100644 new mode 100755 index fdcb808..4f85bba --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -30,7 +30,7 @@ # @(#)newvers.sh 8.1 (Berkeley) 4/20/94 # $FreeBSD$ -TYPE="FreeBSD" +TYPE="ElectroBSD" REVISION="11.0" BRANCH="CURRENT" if [ -n "${BRANCH_OVERRIDE}" ]; then -- 2.7.0 From 1ebc3370d5954d1ad222640c6438d5ad048b19d8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 Jan 2016 13:45:32 +0100 Subject: [PATCH 057/213] mdocml: Change OS name used in man page headers to ElectroBSD --- contrib/mdocml/msec.in | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/mdocml/msec.in b/contrib/mdocml/msec.in index 86d0dd8..09f6f36 100644 --- a/contrib/mdocml/msec.in +++ b/contrib/mdocml/msec.in @@ -22,16 +22,16 @@ * Be sure to escape strings. */ -LINE("1", "FreeBSD General Commands Manual") -LINE("2", "FreeBSD System Calls Manual") -LINE("3", "FreeBSD Library Functions Manual") +LINE("1", "ElectroBSD General Commands Manual") +LINE("2", "ElectroBSD System Calls Manual") +LINE("3", "ElectroBSD Library Functions Manual") LINE("3p", "Perl Library Functions Manual") -LINE("4", "FreeBSD Kernel Interfaces Manual") -LINE("5", "FreeBSD File Formats Manual") -LINE("6", "FreeBSD Games Manual") -LINE("7", "FreeBSD Miscellaneous Information Manual") -LINE("8", "FreeBSD System Manager\'s Manual") -LINE("9", "FreeBSD Kernel Developer\'s Manual") +LINE("4", "ElectroBSD Kernel Interfaces Manual") +LINE("5", "ElectroBSD File Formats Manual") +LINE("6", "ElectroBSD Games Manual") +LINE("7", "ElectroBSD Miscellaneous Information Manual") +LINE("8", "ElectroBSD System Manager\'s Manual") +LINE("9", "ElectroBSD Kernel Developer\'s Manual") LINE("X11", "X11 Developer\'s Manual") LINE("X11R6", "X11 Developer\'s Manual") LINE("unass", "Unassociated") -- 2.7.0 From 863145f620d2604b895bf766a06b6e74fb6eb5a1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 Jan 2016 20:24:00 +0100 Subject: [PATCH 058/213] clang: Set CLANG_VENDOR to ElectroBSD --- lib/clang/include/clang/Basic/Version.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/clang/include/clang/Basic/Version.inc b/lib/clang/include/clang/Basic/Version.inc index 5c5b264..501ed9e 100644 --- a/lib/clang/include/clang/Basic/Version.inc +++ b/lib/clang/include/clang/Basic/Version.inc @@ -5,7 +5,7 @@ #define CLANG_VERSION_MINOR 7 #define CLANG_VERSION_PATCHLEVEL 1 -#define CLANG_VENDOR "FreeBSD " +#define CLANG_VENDOR "ElectroBSD " #define CLANG_VENDOR_SUFFIX " 20151225" #define SVN_REVISION "255217" -- 2.7.0 From d8971c82ebd7155961ff1b35a38c29c77f30223c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 24 Feb 2015 19:35:03 +0100 Subject: [PATCH 059/213] Let rc.d/motd work with unames other than FreeBSD --- etc/rc.d/motd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/rc.d/motd b/etc/rc.d/motd index 8256d96..7141aa5e 100755 --- a/etc/rc.d/motd +++ b/etc/rc.d/motd @@ -34,7 +34,7 @@ motd_start() T=`mktemp -t motd` uname -v | sed -e 's,^\([^#]*\) #\(.* [1-2][0-9][0-9][0-9]\).*/\([^\]*\) $,\1 (\3) #\2,' > ${T} - awk '{if (NR == 1) {if ($1 == "FreeBSD") {next} else {print "\n"$0}} else {print}}' < /etc/motd >> ${T} + awk '{if (NR == 1) {if ($1 == "'"$(uname)"'") {next} else {print "\n"$0}} else {print}}' < /etc/motd >> ${T} cmp -s $T /etc/motd || { cp $T /etc/motd -- 2.7.0 From 2a02ea4dc3d17fe5955d8bb9efa907a382b297e1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 Mar 2015 12:07:50 +0100 Subject: [PATCH 060/213] swapon: Accept the "late" option for .eli devices Using the late option makes it possible to additionally use the consumer device as (unencrypted!) dump device without making it unavailable for rc.d/savecore (which runs between rc.d/swap and rc.d/swaplate). Obtained from: ElectroBSD Submitted in: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198598 Submitted at: 2015-03-15 11:37 UTC --- sbin/swapon/swapon.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbin/swapon/swapon.c b/sbin/swapon/swapon.c index f581431..10d0ac9 100644 --- a/sbin/swapon/swapon.c +++ b/sbin/swapon/swapon.c @@ -367,7 +367,8 @@ swap_on_geli_args(const char *mntops) } } else if ((p = strstr(token, "notrim")) == token) { Tflag = " -T "; - } else if (strcmp(token, "sw") != 0) { + } else if (strcmp(token, "sw") != 0 && + strcmp(token, "late") != 0) { warnx("Invalid option: %s", token); free(ops); return (NULL); -- 2.7.0 From 47c3fb29afd3dc84d6e6983ca3f7346513fa7755 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 18 Apr 2015 19:47:47 +0200 Subject: [PATCH 061/213] gmountver: Debug WIP --- sys/geom/mountver/g_mountver.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sys/geom/mountver/g_mountver.c b/sys/geom/mountver/g_mountver.c index eafccc8..5a346d0 100644 --- a/sys/geom/mountver/g_mountver.c +++ b/sys/geom/mountver/g_mountver.c @@ -48,7 +48,7 @@ __FBSDID("$FreeBSD$"); SYSCTL_DECL(_kern_geom); static SYSCTL_NODE(_kern_geom, OID_AUTO, mountver, CTLFLAG_RW, 0, "GEOM_MOUNTVER stuff"); -static u_int g_mountver_debug = 0; +static u_int g_mountver_debug = 2; static u_int g_mountver_check_ident = 1; SYSCTL_UINT(_kern_geom_mountver, OID_AUTO, debug, CTLFLAG_RW, &g_mountver_debug, 0, "Debug level"); @@ -539,7 +539,18 @@ g_mountver_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) return (NULL); cp = LIST_FIRST(&gp->consumer); + /* + if (cp->provider->mediasize != pp->mediasize) { + G_MOUNTVER_DEBUG(2, "Size mismatch " + "(%lu != %lu). Skipping %s.", + cp->provider->mediasize, pp->mediasize, pp->name); + return (NULL); + } + G_MOUNTVER_DEBUG(2, "Sizes match (%lu == %lu)", + cp->provider->mediasize, pp->mediasize); + */ g_attach(cp, pp); + G_MOUNTVER_DEBUG(2, "Checking ident for %s.", pp->name); error = g_mountver_ident_matches(gp); if (error != 0) { g_detach(cp); -- 2.7.0 From f0d372be7b72a4352e9493d2919a0766657c6842 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 18 Apr 2015 19:38:07 +0200 Subject: [PATCH 062/213] gmountver.8: Note that GEOM mount verification has dangerous bugs --- sbin/geom/class/mountver/gmountver.8 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sbin/geom/class/mountver/gmountver.8 b/sbin/geom/class/mountver/gmountver.8 index 4c27a65..ddb9e2b 100644 --- a/sbin/geom/class/mountver/gmountver.8 +++ b/sbin/geom/class/mountver/gmountver.8 @@ -61,6 +61,8 @@ got disconnected - it queues all the I/O requests and waits for the provider to reappear. When that happens, it attaches to it and sends the queued requests. .Pp +At least that's the theory, please note the BUGS section. +.Pp The first argument to .Nm indicates an action to be performed: @@ -119,6 +121,13 @@ If set to 0, .Nm will reattach to the device even if the device reports a different disk ID. .El +.Sh BUGS +The mount verification GEOM class can stall all the disk I/O instead +of just the device it is configured for. +The disk identification check currently has to be turned off for the class +to actually attach reappearing providers. +Obviously this is dangerous. +.El .Sh EXIT STATUS Exit status is 0 on success, and 1 if the command fails. .Sh SEE ALSO -- 2.7.0 From a1eab069916c8748c188ac5b301388770ec780da Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 19 Apr 2015 22:58:49 +0200 Subject: [PATCH 063/213] Default to listening to 127.0.0.1 only --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 4397253..858dc07 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1055,7 +1055,7 @@ main(int argc, char *argv[]) int ch, sfd, tmpsfd; unsigned port; - bindaddr = htonl(INADDR_ANY); + bindaddr = g_gate_str2ip("127.0.0.1"); port = G_GATE_PORT; while ((ch = getopt(argc, argv, "a:hnp:F:R:S:v")) != -1) { switch (ch) { -- 2.7.0 From f933b9147ee5dca06572030f97d4accf3694d672 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 30 Mar 2015 15:24:05 +0200 Subject: [PATCH 064/213] ggatel: Optionally retry in case of failed reads and writes ... after waiting five seconds and reopening the device. This allows to use ggatel as a workaround against USB flakiness which can result in device disconnects that are extremely annoying if ZFS on geli is involved. To prevent data loss if multiple device disappear at the same time and reappear with different names, the disk ident is checked to confirm that the device is the expected one. As a side-effect retrying will not work for files (which have no disk ident). Obtained from: ElectroBSD --- sbin/ggate/ggatel/ggatel.c | 63 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/sbin/ggate/ggatel/ggatel.c b/sbin/ggate/ggatel/ggatel.c index 0b89df0..864f3ae 100644 --- a/sbin/ggate/ggatel/ggatel.c +++ b/sbin/ggate/ggatel/ggatel.c @@ -53,6 +53,7 @@ static const char *path = NULL; static int unit = G_GATE_UNIT_AUTO; static unsigned flags = 0; static int force = 0; +static unsigned retries = 0; static unsigned sectorsize = 0; static unsigned timeout = G_GATE_TIMEOUT; @@ -60,10 +61,10 @@ static void usage(void) { - fprintf(stderr, "usage: %s create [-v] [-o ] " + fprintf(stderr, "usage: %s create [-v] [-o ] [-r ] " "[-s sectorsize] [-t timeout] [-u unit] \n", getprogname()); - fprintf(stderr, " %s rescue [-v] [-o ] <-u unit> " - "\n", getprogname()); + fprintf(stderr, " %s rescue [-v] [-o ] [-r ] " + "<-u unit> \n", getprogname()); fprintf(stderr, " %s destroy [-f] <-u unit>\n", getprogname()); fprintf(stderr, " %s list [-v] [-u unit]\n", getprogname()); exit(EXIT_FAILURE); @@ -85,6 +86,7 @@ g_gatel_serve(int fd) { struct g_gate_ctl_io ggio; size_t bsize; + char ident[DISK_IDENT_SIZE]; if (g_gate_verbose == 0) { if (daemon(0, 0) == -1) { @@ -97,8 +99,15 @@ g_gatel_serve(int fd) ggio.gctl_unit = unit; bsize = sectorsize; ggio.gctl_data = malloc(bsize); + + errno = 0; + if (retries && ioctl(fd, DIOCGIDENT, ident) != 0) { + g_gate_xlog("Failed to get disk ident for %s: %s", path, + strerror(errno)); + } for (;;) { int error; + int retries_left; once_again: ggio.gctl_length = bsize; ggio.gctl_error = 0; @@ -130,6 +139,8 @@ once_again: strerror(error)); } + retries_left = retries; +retry_request: error = 0; switch (ggio.gctl_cmd) { case BIO_READ: @@ -145,6 +156,10 @@ once_again: if (pread(fd, ggio.gctl_data, ggio.gctl_length, ggio.gctl_offset) == -1) { error = errno; + g_gate_log(LOG_ERR, "Failed to read %d" + " bytes from %s: %s", + ggio.gctl_length, path, + strerror(error)); } } break; @@ -153,12 +168,44 @@ once_again: if (pwrite(fd, ggio.gctl_data, ggio.gctl_length, ggio.gctl_offset) == -1) { error = errno; + g_gate_log(LOG_ERR, "Failed to write %d bytes" + " to %s: %s", ggio.gctl_length, path, + strerror(error)); } break; default: error = EOPNOTSUPP; } + if (error && error != EOPNOTSUPP) { + if (retries_left > 0) { + char ident_new[DISK_IDENT_SIZE]; + + close(fd); + retries_left--; + sleep(5); + fd = open(path, g_gate_openflags(flags) | + O_DIRECT | O_FSYNC); + if (fd == -1) { + err(EXIT_FAILURE, "Cannot open %s", + path); + } + if (ioctl(fd, DIOCGIDENT, ident_new) != 0) { + g_gate_xlog("Failed to get disk ", + "ident for %s: %s", path, + strerror(errno)); + } + if (strcmp(ident, ident_new) != 0) { + g_gate_xlog("Disk ident for %s " + "changed from %s to %s. Reuse " + "could cause data loss.", path, + ident, ident_new); + } + g_gate_log(LOG_ERR, "Retrying after reopening " + "%s (%s)", path, ident); + goto retry_request; + } + } ggio.gctl_error = error; g_gate_ioctl(G_GATE_CMD_DONE, &ggio); } @@ -230,7 +277,7 @@ main(int argc, char *argv[]) for (;;) { int ch; - ch = getopt(argc, argv, "fo:s:t:u:v"); + ch = getopt(argc, argv, "fo:r:s:t:u:v"); if (ch == -1) break; switch (ch) { @@ -253,6 +300,14 @@ main(int argc, char *argv[]) "Invalid argument for '-o' option."); } break; + case 'r': + if (action != CREATE && action != RESCUE) + usage(); + errno = 0; + retries = strtoul(optarg, NULL, 10); + if (retries == 0 && errno != 0) + errx(EXIT_FAILURE, "Invalid retry count."); + break; case 's': if (action != CREATE) usage(); -- 2.7.0 From e4df85cf777b22491b1099e686947f534478b630 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 26 Apr 2015 17:54:14 +0200 Subject: [PATCH 065/213] ggatel.8: Document the shiny new -r option --- sbin/ggate/ggatel/ggatel.8 | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/sbin/ggate/ggatel/ggatel.8 b/sbin/ggate/ggatel/ggatel.8 index f2eea7c..7612b75 100644 --- a/sbin/ggate/ggatel/ggatel.8 +++ b/sbin/ggate/ggatel/ggatel.8 @@ -35,6 +35,7 @@ .Cm create .Op Fl v .Op Fl o Cm ro | wo | rw +.Op Fl r Ar retries .Op Fl s Ar sectorsize .Op Fl t Ar timeout .Op Fl u Ar unit @@ -51,6 +52,7 @@ .Cm rescue .Op Fl v .Op Fl o Cm ro | wo | rw +.Op Fl r Ar retries .Fl u Ar unit .Ar path .Sh DESCRIPTION @@ -102,6 +104,19 @@ or read-write .Pq Cm rw . Default is .Cm rw . +.It Fl r Ar retries +Number of times a failed request should be retried before forwarding +the error to the kernel. +Between retries, +.Nm ggatel +waits for five seconds and reopens the device in case it temporarily +disappeared. +The reopened device is only used if the disk identification did not +change. +This option is useful when using unreliable USB devices as geli +consumer (as long as the device loss does not cause the USB +stack to deadlock). +By default failed requests are not retried. .It Fl s Ar sectorsize Sector size for .Nm ggate @@ -144,6 +159,52 @@ umount /secret gbde detach ggate5 ggatel destroy -u 5 .Ed + +Scrub a pool on an USB device that occasionally disappears: +.Bd -literal -offset indent +$ glabel list da0 +Geom name: da0 +Providers: +1. Name: label/extreme + Mediasize: 4023385600 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + secoffset: 0 + offset: 0 + seclength: 7858175 + length: 4023385600 + index: 0 +Consumers: +1. Name: da0 + Mediasize: 4023386112 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + +$ sudo ggatel create -r 2 /dev/da0 +ggate0 +$ glabel list da0 +glabel: No such geom: da0. +glabel list ggate0 +Geom name: ggate0 +Providers: +1. Name: label/extreme + Mediasize: 4023385600 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + secoffset: 0 + offset: 0 + seclength: 7858175 + length: 4023385600 + index: 0 +Consumers: +1. Name: ggate0 + Mediasize: 4023386112 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + +$ zogftw import extreme +$ sudo zpool scrub extreme +.Ed .Sh SEE ALSO .Xr geom 4 , .Xr gbde 8 , -- 2.7.0 From 190e3b9c93e0b3e7db8ddd9c25c6c9662bac0bfd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:48:20 +0200 Subject: [PATCH 066/213] g_multipath: Add sysctl to disable tasting --- sys/geom/multipath/g_multipath.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sys/geom/multipath/g_multipath.c b/sys/geom/multipath/g_multipath.c index 6644532..bf40ed8 100644 --- a/sys/geom/multipath/g_multipath.c +++ b/sys/geom/multipath/g_multipath.c @@ -58,6 +58,9 @@ SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW, static u_int g_multipath_exclusive = 1; SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW, &g_multipath_exclusive, 0, "Exclusively open providers"); +static u_int g_multipath_enable_tasting = 1; +SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, taste, CTLFLAG_RW, + &g_multipath_enable_tasting, 0, "Enable multipath tasting. May cause conflicts."); static enum { GKT_NIL, @@ -798,6 +801,9 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) g_topology_assert(); + if (g_multipath_enable_tasting == 0) + return (NULL); + gp = g_new_geomf(mp, "multipath:taste"); gp->start = g_multipath_start; gp->access = g_multipath_access; -- 2.7.0 From d1e02048540abddc41e862a8342450b1dba5196b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 6 May 2015 14:04:57 +0200 Subject: [PATCH 067/213] zvol.c: Fix comment --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c index 5eb9df1..58c6bba 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c @@ -2269,7 +2269,7 @@ zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr, int *rvalp) /* * If the caller really wants synchronous writes, and - * can't wait for them, don't return until the write + * can wait for them, don't return until the write * is done. */ if (df.df_flags & DF_WAIT_SYNC) { -- 2.7.0 From 140f7562e2e987ca3dbd5d8a9a896edc65be5662 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 May 2015 14:32:31 +0200 Subject: [PATCH 068/213] motd: Customize for ElectroBSD --- etc/motd | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/etc/motd b/etc/motd index 4dc41a7..29a81b8 100644 --- a/etc/motd +++ b/etc/motd @@ -1,21 +1,8 @@ -FreeBSD ?.?.? (UNKNOWN) +ElectroBSD ?.?.? (UNKNOWN) -Welcome to FreeBSD! +Welcome to ElectroBSD! -Release Notes, Errata: https://www.FreeBSD.org/releases/ -Security Advisories: https://www.FreeBSD.org/security/ -FreeBSD Handbook: https://www.FreeBSD.org/handbook/ -FreeBSD FAQ: https://www.FreeBSD.org/faq/ -Questions List: https://lists.FreeBSD.org/mailman/listinfo/freebsd-questions/ -FreeBSD Forums: https://forums.FreeBSD.org/ - -Documents installed with the system are in the /usr/local/share/doc/freebsd/ -directory, or can be installed later with: pkg install en-freebsd-doc -For other languages, replace "en" with a language code like de or fr. - -Show the version of FreeBSD installed: freebsd-version ; uname -a -Please include that output and any error messages when posting questions. -Introduction to manual pages: man man -FreeBSD directory layout: man hier +For details see: +https://www.fabiankeil.de/gehacktes/electrobsd/ Edit /etc/motd to change this login announcement. -- 2.7.0 From 298530adb6bcba89408f04e0641b1ad7c82ec34c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 14:18:43 +0200 Subject: [PATCH 069/213] release: Load usb modules through loader.conf so 'memstick' works with ELECTRO_BEER --- release/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/Makefile b/release/Makefile index ba1ca80..1570aa3 100644 --- a/release/Makefile +++ b/release/Makefile @@ -238,6 +238,9 @@ dvd: packagesystem echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo debug.witness.trace=0 >> ${.TARGET}/etc/sysctl.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf + for module in usb usb_quirk ehci umass; do \ + echo $${module}_load=\"YES\" >> ${.TARGET}/boot/loader.conf + done cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.7.0 From a80feb412734a0b2fa461bcb27586e94f44b7fd8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:13:51 +0200 Subject: [PATCH 070/213] brand-fbsd.4th: Change OS name in banner to ElectroBSD While at it, suggest to resist unlawful police activities (German). --- sys/boot/forth/brand-fbsd.4th | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sys/boot/forth/brand-fbsd.4th b/sys/boot/forth/brand-fbsd.4th index 9cd017f..03392ca 100644 --- a/sys/boot/forth/brand-fbsd.4th +++ b/sys/boot/forth/brand-fbsd.4th @@ -32,15 +32,17 @@ 1+ \ increase y for next time we're called ; -: brand ( x y -- ) \ "FreeBSD" [wide] logo in B/W (7 rows x 42 columns) +: brand ( x y -- ) \ "ElectroBSD" [wide] logo in B/W - s" ______ ____ _____ _____ " brand+ - s" | ____| | _ \ / ____| __ \ " brand+ - s" | |___ _ __ ___ ___ | |_) | (___ | | | |" brand+ - s" | ___| '__/ _ \/ _ \| _ < \___ \| | | |" brand+ - s" | | | | | __/ __/| |_) |____) | |__| |" brand+ - s" | | | | | | || | | |" brand+ - s" |_| |_| \___|\___||____/|_____/|_____/ " brand+ + s" ______ _ _ ____ _____ _____" brand+ + s" | ____| | | | | _ \ / ____| __ \" brand+ + s" | |__ | | ___ ___| |_ _ __ ___ | |_) | (___ | | | |" brand+ + s" | __| | |/ _ \/ __| __| '__/ _ \| _ < \___ \| | | |" brand+ + s" | |____| | __/ (__| |_| | | (_) | |_) |____) | |__| |" brand+ + s" |______|_|\___|\___|\__|_| \___/|____/|_____/|_____/" brand+ + s" Polizei-Willkuer in Deinem Land? Das erfordert Widerstand!" brand+ + s" Zu Risiken und Nebenwirkungen fragen Sie Ihren Anwalt oder" brand+ + s" die Rote Hilfe." brand+ 2drop ; -- 2.7.0 From d62b553226631fdb3d254199ab4610a993d4e521 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:25:56 +0200 Subject: [PATCH 071/213] beastie.4th: Think of the children and default to showing beastie instead of the sex toy --- sys/boot/forth/beastie.4th | 4 ++-- sys/boot/forth/beastie.4th.8 | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sys/boot/forth/beastie.4th b/sys/boot/forth/beastie.4th index 752cce2..6a28605 100644 --- a/sys/boot/forth/beastie.4th +++ b/sys/boot/forth/beastie.4th @@ -64,9 +64,9 @@ variable logoY s" loader_logo" getenv dup -1 = over 0= or if dup 0= if 2drop else drop then \ getenv result unused loader_color? if - s" try-include /boot/logo-orb.4th" + s" try-include /boot/logo-beastie.4th" else - s" try-include /boot/logo-orbbw.4th" + s" try-include /boot/logo-beastiebw.4th" then else 2drop ( c-addr/u -- ) \ getenv result unused diff --git a/sys/boot/forth/beastie.4th.8 b/sys/boot/forth/beastie.4th.8 index 9f77d5d..3ea13c1 100644 --- a/sys/boot/forth/beastie.4th.8 +++ b/sys/boot/forth/beastie.4th.8 @@ -106,10 +106,9 @@ The environment variables that effect its behavior are: Selects the desired logo in the beastie boot menu. Possible values are: .Dq Li fbsdbw , .Dq Li beastie , -.Dq Li beastiebw , +.Dq Li beastiebw (default) , .Dq Li orb , -.Dq Li orbbw -(default), and +.Dq Li orbbw , and .Dq Li none . .It Va loader_logo_x Sets the desired column position of the logo. Default is 46. -- 2.7.0 From d6a46da21fc634d0a0ecea36dd34677da72be755 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:39:56 +0200 Subject: [PATCH 072/213] menu.4th: Reduce visual noise by ditching the welcome text --- sys/boot/forth/menu.4th | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/forth/menu.4th b/sys/boot/forth/menu.4th index 9127565..5db2163 100644 --- a/sys/boot/forth/menu.4th +++ b/sys/boot/forth/menu.4th @@ -470,7 +470,7 @@ also menu-infrastructure definitions \ Print the frame caption at (x,y) s" loader_menu_title" getenv dup -1 = if - drop s" Welcome to FreeBSD" + drop s" " then TRUE ( use default alignment ) s" loader_menu_title_align" getenv dup -1 <> if -- 2.7.0 From 81fd9e979b0ddc2b1b4d10e34407445011180291 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 17:40:22 +0200 Subject: [PATCH 073/213] logo-beastiebw: Change the fork to a toilet brush --- sys/boot/forth/logo-beastiebw.4th | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sys/boot/forth/logo-beastiebw.4th b/sys/boot/forth/logo-beastiebw.4th index 197099c..bce43f5 100644 --- a/sys/boot/forth/logo-beastiebw.4th +++ b/sys/boot/forth/logo-beastiebw.4th @@ -33,27 +33,27 @@ 1+ \ increase y for next time we're called ; -: logo ( x y -- ) \ B/W BSD mascot (19 rows x 34 columns) +: logo ( x y -- ) \ B/W BSD mascot with toilet brush - s" , ," logo+ - s" /( )`" logo+ - s" \ \___ / |" logo+ - s" /- _ `-/ '" logo+ - s" (/\/ \ \ /\" logo+ - s" / / | ` \" logo+ - s" O O ) / |" logo+ - s" `-^--'`< '" logo+ - s" (_.) _ ) /" logo+ - s" `.___/` /" logo+ - s" `-----' /" logo+ - s" <----. __ / __ \" logo+ - s" <----|====O)))==) \) /====|" logo+ - s" <----' `--' `.__,' \" logo+ - s" | |" logo+ - s" \ / /\" logo+ - s" ______( (_ / \______/" logo+ - s" ,' ,-----' |" logo+ - s" `--{__________)" logo+ + s" , ," logo+ + s" /( )`" logo+ + s" \ \___ / |" logo+ + s" /- _ `-/ '" logo+ + s" (/\/ \ \ /\" logo+ + s" / / | ` \" logo+ + s" O O ) / |" logo+ + s" `-^--'`< '" logo+ + s" (_.) _ ) /" logo+ + s" `.___/` /" logo+ + s" `-----' /" logo+ + s" ###### __ / __ \" logo+ + s" ######===O)))==) \) /====|" logo+ + s" ###### `--' `.__,' \" logo+ + s" | |" logo+ + s" \ / /\" logo+ + s" ______( (_ / \___/" logo+ + s" ,' ,-----' |" logo+ + s" `--{__________)" logo+ 2drop ; -- 2.7.0 From 17c0d7a239bbaaa7743b94c4f9cdb038fac2b651 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 18:19:19 +0200 Subject: [PATCH 074/213] logo-beastie.4th: Replace fork with toilet brush --- sys/boot/forth/logo-beastie.4th | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sys/boot/forth/logo-beastie.4th b/sys/boot/forth/logo-beastie.4th index 671eb5e..8441c9d 100644 --- a/sys/boot/forth/logo-beastie.4th +++ b/sys/boot/forth/logo-beastie.4th @@ -35,27 +35,27 @@ 1+ \ increase y for next time we're called ; -: logo ( x y -- ) \ color BSD mascot (19 rows x 34 columns) +: logo ( x y -- ) \ color BSD mascot with toilet brush - s" @[31m, ," logo+ - s" /( )`" logo+ - s" \ \___ / |" logo+ - s" /- @[m_@[31m `-/ '" logo+ - s" (@[m/\/ \@[31m \ /\" logo+ - s" @[m/ / |@[31m ` \" logo+ - s" @[34mO O @[m) @[31m/ |" logo+ - s" @[m`-^--'@[31m`< '" logo+ - s" (_.) _ ) /" logo+ - s" `.___/` /" logo+ - s" `-----' /" logo+ - s" @[33m<----.@[31m __ / __ \" logo+ - s" @[33m<----|====@[31mO)))@[33m==@[31m) \) /@[33m====|" logo+ - s" @[33m<----'@[31m `--' `.__,' \" logo+ - s" | |" logo+ - s" \ / /\" logo+ - s" @[36m______@[31m( (_ / \______/" logo+ - s" @[36m,' ,-----' |" logo+ - s" `--{__________)@[m" logo+ + s" @[31m, ," logo+ + s" /( )`" logo+ + s" \ \___ / |" logo+ + s" /- @[m_@[31m `-/ '" logo+ + s" (@[m/\/ \@[31m \ /\" logo+ + s" @[m/ / |@[31m ` \" logo+ + s" @[34mO O @[m) @[31m/ |" logo+ + s" @[m`-^--'@[31m`< '" logo+ + s" (_.) _ ) /" logo+ + s" `.___/` /" logo+ + s" `-----' /" logo+ + s" @[37m######@[31m __ / __ \" logo+ + s" @[37m######====@[31mO)))@[37m==@[31m) \) /@[37m====|" logo+ + s" @[37m######@[31m `--' `.__,' \" logo+ + s" | |" logo+ + s" \ / /\" logo+ + s" @[36m______@[31m( (_ / \_____/" logo+ + s" @[36m,' ,-----' |" logo+ + s" `--{__________)@[m" logo+ 2drop ; -- 2.7.0 From e51e314ed66bb0206be4e292694a7d56a8b25c7d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 13:24:26 +0200 Subject: [PATCH 075/213] newsyslog.conf: Do not give world read permissions and don't compress stuff --- etc/newsyslog.conf | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/etc/newsyslog.conf b/etc/newsyslog.conf index ab595ca..a409493 100644 --- a/etc/newsyslog.conf +++ b/etc/newsyslog.conf @@ -10,34 +10,28 @@ # # The 'flags' field is one or more of the letters: BCDGJNUXZ or a '-'. # -# Note: some sites will want to select more restrictive protections than the -# defaults. In particular, it may be desirable to switch many of the 644 -# entries to 640 or 600. For example, some sites will consider the -# contents of maillog, messages, and lpd-errs to be confidential. In the -# future, these defaults may change to more conservative ones. -# # logfilename [owner:group] mode count size when flags [/pid_file] [sig_num] -/var/log/all.log 600 7 * @T00 J -/var/log/amd.log 644 7 100 * J -/var/log/auth.log 600 7 100 @0101T JC -/var/log/console.log 600 5 100 * J -/var/log/cron 600 3 100 * JC -/var/log/daily.log 640 7 * @T00 JN -/var/log/debug.log 600 7 100 * JC -/var/log/init.log 644 3 100 * J -/var/log/kerberos.log 600 7 100 * J -/var/log/lpd-errs 644 7 100 * JC -/var/log/maillog 640 7 * @T00 JC -/var/log/messages 644 5 100 @0101T JC -/var/log/monthly.log 640 12 * $M1D0 JN -/var/log/pflog 600 3 100 * JB /var/run/pflogd.pid -/var/log/ppp.log root:network 640 3 100 * JC -/var/log/devd.log 644 3 100 * JC -/var/log/security 600 10 100 * JC +/var/log/all.log 600 7 * @T00 +/var/log/amd.log 640 7 100 * +/var/log/auth.log 600 7 100 @0101T C +/var/log/console.log 600 5 100 * +/var/log/cron 600 3 100 * C +/var/log/daily.log 640 7 * @T00 N +/var/log/debug.log 600 7 100 * C +/var/log/init.log 640 3 100 * +/var/log/kerberos.log 600 7 100 * +/var/log/lpd-errs 640 7 100 * C +/var/log/maillog 640 7 * @T00 C +/var/log/messages 640 5 100 @0101T C +/var/log/monthly.log 640 12 * $M1D0 N +/var/log/pflog 600 3 100 * B /var/run/pflogd.pid +/var/log/ppp.log root:network 640 3 100 * C +/var/log/devd.log 640 3 100 * C +/var/log/security 600 10 100 * C /var/log/sendmail.st 640 10 * 168 BN -/var/log/utx.log 644 3 * @01T05 B -/var/log/weekly.log 640 5 * $W6D0 JN -/var/log/xferlog 600 7 100 * JC +/var/log/utx.log 640 3 * @01T05 B +/var/log/weekly.log 640 5 * $W6D0 N +/var/log/xferlog 600 7 100 * C /etc/newsyslog.conf.d/* /usr/local/etc/newsyslog.conf.d/* -- 2.7.0 From 45bfed7b19fe97f6fd30ec3a91d856139ddd29b4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 10:57:01 +0200 Subject: [PATCH 076/213] rc.d/jail: Remove obnoxious warning about 'obsolete' jail_* variables They will not be removed from ElectroBSD until a replacement exists that is usable with shell scripts like ezjail without jumping through lots of hoops. --- etc/rc.d/jail | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/rc.d/jail b/etc/rc.d/jail index fa0bc46..d71cf22 100755 --- a/etc/rc.d/jail +++ b/etc/rc.d/jail @@ -14,7 +14,6 @@ name="jail" rcvar="jail_enable" start_cmd="jail_start" -start_postcmd="jail_warn" stop_cmd="jail_stop" config_cmd="jail_config" console_cmd="jail_console" -- 2.7.0 From c7dd78df0404d47393b06e2f32bd0d9976f47421 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 16:04:23 +0200 Subject: [PATCH 077/213] Don't build rcs papers until they built reproducible. --- share/doc/psd/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/share/doc/psd/Makefile b/share/doc/psd/Makefile index 6b6d9cd..9e0eed2 100644 --- a/share/doc/psd/Makefile +++ b/share/doc/psd/Makefile @@ -20,7 +20,6 @@ SUBDIR= title \ 05.sysman \ 06.Clang \ 12.make \ - 13.rcs \ 15.yacc \ 16.lex \ 17.m4 \ -- 2.7.0 From 2f56b2977658fc38b5a50b934f4cc0c0de48a157 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 12:34:11 +0200 Subject: [PATCH 078/213] kern.opts.mk: Disable IPFILTER, SOURCELESS_HOST and SOURCELESS_UCODE by default src.conf(5) will be updated once I find the time to figure out how to use tools/build/options/makeman. --- sys/conf/kern.opts.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sys/conf/kern.opts.mk b/sys/conf/kern.opts.mk index 1a3a1fd..c917dbb 100644 --- a/sys/conf/kern.opts.mk +++ b/sys/conf/kern.opts.mk @@ -33,21 +33,21 @@ __DEFAULT_YES_OPTIONS = \ FORMAT_EXTENSIONS \ INET \ INET6 \ - IPFILTER \ ISCSI \ KERNEL_SYMBOLS \ NETGRAPH \ PF \ - SOURCELESS_HOST \ - SOURCELESS_UCODE \ USB_GADGET_EXAMPLES \ ZFS __DEFAULT_NO_OPTIONS = \ EISA \ FAST_DEPEND \ + IPFILTER \ NAND \ - OFED + OFED \ + SOURCELESS_HOST \ + SOURCELESS_UCODE # Some options are totally broken on some architectures. We disable # them. If you need to enable them on an experimental basis, you -- 2.7.0 From e4c92eb4847edd12073447af223a27bcfac0a343 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 15 May 2015 13:11:57 +0200 Subject: [PATCH 079/213] boot/newvers.sh branding: Change bootprog_name[] to ElectroBSD --- sys/boot/common/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/common/newvers.sh b/sys/boot/common/newvers.sh index ee2ac99..cd7422f 100755 --- a/sys/boot/common/newvers.sh +++ b/sys/boot/common/newvers.sh @@ -40,7 +40,7 @@ u=${USER-root} h=${HOSTNAME-`hostname`} t=`date` #r=`head -n 6 $1 | tail -n 1 | awk -F: ' { print $1 } '` r=`awk -F: ' /^[0-9]\.[0-9]+:/ { print $1; exit }' $1` -echo "char bootprog_name[] = \"FreeBSD/${3} ${2}\";" > $tempfile +echo "char bootprog_name[] = \"ElectroBSD/${3} ${2}\";" > $tempfile echo "char bootprog_rev[] = \"${r}\";" >> $tempfile echo "char bootprog_date[] = \"${t}\";" >> $tempfile echo "char bootprog_maker[] = \"${u}@${h}\";" >> $tempfile -- 2.7.0 From 9715e9b038857357631c0bf4f5b1af83723ad9ed Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 15:59:58 +0200 Subject: [PATCH 080/213] release: Default to not distributing the ports tree Allow to overwrite this with WITH_PORTS --- release/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/Makefile b/release/Makefile index 1570aa3..54b04b5 100644 --- a/release/Makefile +++ b/release/Makefile @@ -23,9 +23,9 @@ # DOCDIR: location of doc tree (default: /usr/doc) # XTRADIR: xtra-bits-dir argument for /mkisoimages.sh # NOPKG: if set, do not distribute third-party packages -# NOPORTS: if set, do not distribute ports tree # NOSRC: if set, do not distribute source tree # NODOC: if set, do not generate release documentation +# WITH_PORTS: if set, distribute ports tree provided it exists # WITH_DVD: if set, generate dvd1.iso # WITH_COMPRESSED_IMAGES: if set, compress installation images with xz(1) # (uncompressed images are not removed) @@ -77,7 +77,7 @@ VOLUME_LABEL= FreeBSD_Install .if !exists(${DOCDIR}) NODOC= true .endif -.if !exists(${PORTSDIR}) +.if !exists(${PORTSDIR}) || !defined(WITH_PORTS) NOPORTS= true .endif -- 2.7.0 From 531062571725ceedcb441331c3d1a7136a935802 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:26:44 +0200 Subject: [PATCH 081/213] sys/boot/common/newvers.sh: Allow to overwrite the date to make boot loader binaries reproducible --- sys/boot/common/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/common/newvers.sh b/sys/boot/common/newvers.sh index cd7422f..8c5e9d6 100755 --- a/sys/boot/common/newvers.sh +++ b/sys/boot/common/newvers.sh @@ -36,7 +36,7 @@ tempfile=$(mktemp tmp.XXXXXX) || exit trap "rm -f $tempfile" EXIT INT TERM LC_ALL=C; export LC_ALL -u=${USER-root} h=${HOSTNAME-`hostname`} t=`date` +u=${USER-root} h=${HOSTNAME-`hostname`} t=${DATE-`date`} #r=`head -n 6 $1 | tail -n 1 | awk -F: ' { print $1 } '` r=`awk -F: ' /^[0-9]\.[0-9]+:/ { print $1; exit }' $1` -- 2.7.0 From 7f28987740b57b5b211983b164f28e5e217fe3aa Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:36:48 +0200 Subject: [PATCH 082/213] sys/conf/newvers.sh: Allow to overwrite the build date embedded into the kernel This is a required step to get reproducible builds. --- sys/conf/newvers.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 4f85bba..8717307 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -101,7 +101,12 @@ v=`cat version` u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} -if [ -n "$SOURCE_DATE_EPOCH" ]; then +if [ -n "${DATE}" ]; then + # SOURCE_DATE_EPOCH was added upstream in r291691 + # but the ElectroBSD build goo is still setting DATE + # which has a different format. + t=${DATE} +elif [ -n "$SOURCE_DATE_EPOCH" ]; then if ! t=`date -r $SOURCE_DATE_EPOCH 2>/dev/null`; then echo "Invalid SOURCE_DATE_EPOCH" >&2 exit 1 -- 2.7.0 From c0eeb16f61bb85a2ca81f15b1d2a175d85a03e93 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:10:44 +0200 Subject: [PATCH 083/213] amd64/make-memstick.sh: Create more 'predictable' images According to the mkimg man page, the "-y option is used for testing purposes only and is not to be used in production", but it feels good to be a gangsta. --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 52e17cc..d9590d1 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -38,6 +38,6 @@ fi rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local -mkimg -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.7.0 From f264a4a6829db9b056c5ebb240eff7209075042a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:17:14 +0200 Subject: [PATCH 084/213] release/Makefile: Don't create matroshka src tarballs that contain other tarballs ... if DESTDIR isn't set to a reasonable value. While at it, exclude *.orig files as well. --- release/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index 54b04b5..63a148a 100644 --- a/release/Makefile +++ b/release/Makefile @@ -145,7 +145,9 @@ src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist usr/src | \ + --exclude .git --exclude @ --exclude usr/src/release/dist \ + --exclude "usr/src/release/*.txz" \ + --exclude "usr/src/release/*.orig" usr/src | \ ${XZ_CMD} > ${.OBJDIR}/src.txz ports.txz: -- 2.7.0 From 2d87927b60b31527e8cb39c4ad0b84d0b0019279 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 18:45:24 +0200 Subject: [PATCH 085/213] release: Use a hack to recreate dist tarballs with reproducible timestamps --- release/Makefile | 3 ++ release/scripts/tar-time-reset.sh | 64 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100755 release/scripts/tar-time-reset.sh diff --git a/release/Makefile b/release/Makefile index 63a148a..456631d 100644 --- a/release/Makefile +++ b/release/Makefile @@ -265,6 +265,9 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} + for tarball in *.txz; do \ + sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ + done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh new file mode 100755 index 0000000..8186fd8 --- /dev/null +++ b/release/scripts/tar-time-reset.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# This script resets the timestamps in a given tarfile to hopefully make +# it reproducible. This is a rather wasteful approach, but works for now. +# +# A better solution would be to patch bsdtar to optionally use a fixed +# time (without having to clown around with mtree specs). +# +########################################################################## + +main() { + local tarfile="${1}" \ + tempdir mtree_spec + + if [ $# -ne 1 ]; then + echo "$0 /path/to/tarfile" + exit 1 + fi + + if [ -z "${tarfile}" ]; then + echo "No tar file given" + return 1 + fi + tarfile="$(realpath "$tarfile")" + + tempdir=$(mktemp -d) || return 1 + mtree_spec=$(mktemp) || return 1 + + echo "Extracting tarfile ${tarfile}" + (cd "${tempdir}" && tar xvf "${tarfile}") || return 1 + + echo "Ditching original tarfile ${tarfile}" + rm "${tarfile}" + + echo "Creating mtree spec in ${mtree_spec}" + (cd "${tempdir}" && mtree -L -c -k time) | \ + sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${mtree_spec}" + + echo "Creating tarfile ${tarfile}" + (cd "${tempdir}" && tar acLvf "${tarfile}" @"${mtree_spec}") || return 1 + + echo "Ditching ${tempdir}" + rm -r "${tempdir}" || return 1 + echo "Ditching ${mtree_spec}" + rm "${mtree_spec}" || return 1 +} + +main "${@}" -- 2.7.0 From e79bf6af011448553cfa898db26e6ecfa9645880 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 May 2015 16:06:01 +0200 Subject: [PATCH 086/213] release/amd64/make-memstick.sh: Use reproducible timestamps for the makefs image --- release/amd64/make-memstick.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index d9590d1..a5d88e1 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -30,13 +30,15 @@ fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -makefs -B little -o label=FreeBSD_Install ${2}.part ${1} +mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 fi rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local +rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.7.0 From c31ade2023db1cdd5352b0d97858853657373d53 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 14:25:17 +0200 Subject: [PATCH 087/213] Remove build timestamps from ntp* binaries again This was already done by r195626 a couple of years ago apparently the change got lost while importing an update from the vendor. --- contrib/ntp/scripts/build/mkver.in | 2 -- 1 file changed, 2 deletions(-) mode change 100644 => 100755 contrib/ntp/scripts/build/mkver.in diff --git a/contrib/ntp/scripts/build/mkver.in b/contrib/ntp/scripts/build/mkver.in old mode 100644 new mode 100755 index 3aef1c8..badd2cd --- a/contrib/ntp/scripts/build/mkver.in +++ b/contrib/ntp/scripts/build/mkver.in @@ -15,8 +15,6 @@ case "@VER_SUFFIX@" in *) ConfStr="${ConfStr}-@VER_SUFFIX@" ;; esac -ConfStr="$ConfStr `LC_TIME=C TZ=UTC date`" - if [ ! -f .version ]; then echo 0 > .version fi -- 2.7.0 From 1e114866ea57544c935c78be00c69f54b659b711 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 12:43:25 +0200 Subject: [PATCH 088/213] usr.sbin/ntp: Allow to set MKREPRO_DATE and MKREPRO_TIME to get reproducible builds --- usr.sbin/ntp/libntp/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/usr.sbin/ntp/libntp/Makefile b/usr.sbin/ntp/libntp/Makefile index 6a58cdb..63899d9 100644 --- a/usr.sbin/ntp/libntp/Makefile +++ b/usr.sbin/ntp/libntp/Makefile @@ -82,6 +82,13 @@ CFLAGS+= -I${.CURDIR}/../../../contrib/ntp/include \ CFLAGS+= -DHAVE_BSD_NICE -DHAVE_STDINT_H +.if defined(MKREPRO_DATE) +CFLAGS+= -DMKREPRO_DATE="\"${MKREPRO_DATE}\"" +.endif +.if defined(MKREPRO_TIME) +CFLAGS+= -DMKREPRO_TIME="\"${MKREPRO_TIME}\"" +.endif + CLEANFILES+= .version version.c version.c: -- 2.7.0 From baa2e6792816dfafba1e5820b89699130d0fdae7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 May 2015 11:33:50 +0200 Subject: [PATCH 089/213] release: Allow to build the mtree spec for mergemaster reproducible This relies on NetBSD mtree which has been the default for a while now. --- release/scripts/mm-mtree.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release/scripts/mm-mtree.sh b/release/scripts/mm-mtree.sh index 4499c10..d5229be 100755 --- a/release/scripts/mm-mtree.sh +++ b/release/scripts/mm-mtree.sh @@ -146,8 +146,11 @@ find ${TEMPROOT} -type f -size 0 -delete 2>/dev/null find -d ${TEMPROOT} -type d -empty -delete 2>/dev/null # Build the mtree database in a temporary location. +# The second mtree call is used to get a reproducible result +# without embedded hostname, user name and creation timestamp. MTREENEW=`mktemp -t mergemaster.mtree` -mtree -ci -p ${TEMPROOT} -k size,md5digest > ${MTREENEW} 2>/dev/null +mtree -ci -p ${TEMPROOT} -k size,md5digest 2>/dev/null | \ + mtree -C -k all > ${MTREENEW} if [ -s "${MTREENEW}" ]; then echo "*** Saving mtree database for future upgrades" -- 2.7.0 From 3b17486e2d0a15cecb8f3b301ab86b6ec254d87f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 12:07:25 +0200 Subject: [PATCH 090/213] Make reproducing builds more convenient ... by setting the various variables based on the environment variable REPRO_SEED. --- Makefile.inc1 | 14 +++++++--- release/Makefile | 2 ++ share/mk/src.reproducible-build.mk | 54 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 share/mk/src.reproducible-build.mk diff --git a/Makefile.inc1 b/Makefile.inc1 index 2359c75..053425d 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -48,6 +48,8 @@ .error "Both TARGET and TARGET_ARCH must be defined." .endif +.include "share/mk/src.reproducible-build.mk" + LOCALBASE?= /usr/local # Cross toolchain changes must be in effect before bsd.compiler.mk @@ -779,17 +781,23 @@ WMAKE_TGTS+= build32 buildworld: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue .ORDER: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue -buildworld_prologue: +buildworld_prologue: reproducible_build_hint @echo "--------------------------------------------------------------" @echo ">>> World build started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" -buildworld_epilogue: +buildworld_epilogue: reproducible_build_hint @echo @echo "--------------------------------------------------------------" @echo ">>> World build completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" +reproducible_build_hint: + @echo "--------------------------------------------------------------" + @echo ">>> To reproduce this build:" + @echo ">>> export REPRO_SEED=$${REPRO_SEED}" + @echo "--------------------------------------------------------------" + # # We need to have this as a target because the indirection between Makefile # and Makefile.inc1 causes the correct PATH to be used, rather than a @@ -1154,7 +1162,7 @@ ${WMAKE_TGTS:N_worldtmp:Nbuild32} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY # # Builds all kernels defined by BUILDKERNELS. # -buildkernel: .MAKE .PHONY +buildkernel: .MAKE .PHONY reproducible_build_hint .if empty(BUILDKERNELS:Ndummy) @echo "ERROR: Missing kernel configuration file(s) (${KERNCONF})."; \ false diff --git a/release/Makefile b/release/Makefile index 456631d..367dd03 100644 --- a/release/Makefile +++ b/release/Makefile @@ -36,6 +36,8 @@ # TARGET/TARGET_ARCH: architecture of built release # +.include "../share/mk/src.reproducible-build.mk" + WORLDDIR?= ${.CURDIR}/.. PORTSDIR?= /usr/ports DOCDIR?= /usr/doc diff --git a/share/mk/src.reproducible-build.mk b/share/mk/src.reproducible-build.mk new file mode 100644 index 0000000..dc4f4df --- /dev/null +++ b/share/mk/src.reproducible-build.mk @@ -0,0 +1,54 @@ +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# Make the build reproducible by exporting a bunch of variables, +# potentionally using an already-set REPRO_SEED as input. +# +# The variable names are somewhat stupid, mostly because we +# are using existing ones. +# +########################################################################## + +TZ= "UTC" +.export TZ + +.if ! defined(REPRO_SEED) +# XXX: Currently we don't add the kernel version number to the repro +# seed because we expect a clean object tree in which case it +# will reproducible be 0. If the object tree of a the +# build-to-reproduce was actually unclean, KERNEL_VERSION_NUMBER +# has to be set to a matching value. +REPRO_SEED!= echo $$(id -un):$$(hostname):$$(date +%s) +.export REPRO_SEED +.else +.if ! defined(KERNEL_VERSION_NUMBER) +KERNEL_VERSION_NUMBER=0 +.export KERNEL_VERSION_NUMBER +.endif +.endif + +USER!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 1 +HOSTNAME!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 2 +EPOCH_DATE!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 3 + +DATE!= date -r ${EPOCH_DATE} +# These two probably are no longer necessary after r285701 +MKREPRO_DATE!= date -r ${EPOCH_DATE} +"%b %d %Y" +MKREPRO_TIME!= date -r ${EPOCH_DATE} +%H:%M:%S + +.for v in REPRO_SEED USER HOSTNAME EPOCH_DATE DATE MKREPRO_DATE MKREPRO_TIME +.export $v +.endfor -- 2.7.0 From b3c55fcb157c4b276fe177eddfc2f7d043a21687 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 23 May 2015 20:13:18 +0200 Subject: [PATCH 091/213] contrib/groff/mdate.sh: Use ${EPOCH_DATE} when set While this is silly, using the time the man pages have been last checked out from the VCS is silly as well, so this commit doesn't increase the total amount of sillyness in the system. Also we are talking about GNU roff here, so hopefully this stuff will be garbage-collected in the near future. --- contrib/groff/mdate.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/groff/mdate.sh b/contrib/groff/mdate.sh index 4a26e6e..615abc7 100755 --- a/contrib/groff/mdate.sh +++ b/contrib/groff/mdate.sh @@ -1,12 +1,17 @@ #! /bin/sh # Print the modification date of $1 `nicely'. +# If ${EPOCH_DATE} is set, it will be used instead. # Don't want foreign dates. LANGUAGE= LC_ALL=C; export LC_ALL +if [ -n "${EPOCH_DATE}" ]; then + date -r "${EPOCH_DATE}" +"%d %B %Y" + exit 0 +fi (date; if ls -L /dev/null 1>/dev/null 2>&1; then ls -L -l $1; else ls -l $1; fi -- 2.7.0 From dab0c5e38fe19265bb5eff8a5c3c5e1f5a3c70db Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 May 2015 18:01:31 +0200 Subject: [PATCH 092/213] Fake modification time smarter for everything but EXTRA_PACKAGES While at it, remove duplicated slashes in the METALOG as they result in missing files. --- Makefile.inc1 | 15 +++++++++++++++ release/Makefile | 6 +++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 053425d..ae291ed 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1010,6 +1010,13 @@ distributeworld installworld: _installcheck_world find ${DESTDIR}/${DISTDIR}/${dist} -mindepth 1 -empty -delete .endfor .if defined(NO_ROOT) + @# Post process METALOG: add fake timestamps and, if necessary, + @# remove duplicated slashes which can occur if DISTDIR is undefined. + @# Keeping them results in missing files in the distribution tarballs. +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -e 's@//@/@' -i '.bak' ${METALOG} +.endif .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @@ -1292,6 +1299,10 @@ distributekernel distributekernel.debug: packagekernel: .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.meta | \ @@ -1303,6 +1314,10 @@ packagekernel: ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz .if ${BUILDKERNELS:[#]} > 1 && !defined(NO_INSTALLEXTRAKERNELS) .for _kernel in ${BUILDKERNELS:[2..-1]} +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ diff --git a/release/Makefile b/release/Makefile index 367dd03..4c2b530 100644 --- a/release/Makefile +++ b/release/Makefile @@ -56,6 +56,10 @@ TARGET_ARCH= ${TARGET} IMAKE= ${MAKE} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET} DISTDIR= dist +# Enable mtree spec usage so we can fake the modification time. +NO_ROOT=1 +.export NO_ROOT + # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -267,7 +271,7 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in *.txz; do \ + for tarball in ${EXTRA_PACKAGES}; do \ sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST -- 2.7.0 From ab6e6d1ba66f82387c6e5fec160b381202d129e9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 10:27:48 +0200 Subject: [PATCH 093/213] Add image-checksum.sh ... which calculates a checksum of the reproducible parts of an memstick image. --- release/scripts/image-checksum.sh | 156 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100755 release/scripts/image-checksum.sh diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh new file mode 100755 index 0000000..6f79a81 --- /dev/null +++ b/release/scripts/image-checksum.sh @@ -0,0 +1,156 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# image-checksum.sh /path/to/memstick.img +# +# Unfortunately the memstick target currently does not create reproducible +# ElectroBSD images due to unreproducible differences in the file system +# layer of the data partition. +# +# To be able to (sort of) compare memstick images anyway, this script +# produces a "partial image checksum" that is based on the partition layout, +# the checksum of the boot code partition and an mtree spec of the data +# partition which includes checksums, sizes and timestamps for all the +# files. +# +# A memstick image whose "partial checksum" matches the one of another +# image can be totally considered to be nearly as trustworthy. Obviously +# that's a somewhat worthless property, it is thus recommended that you +# rebuild the potentionally malicious image using a trusted operating +# system first. After you've done this, potentionally malicious differences +# in the unchecked parts should be gone. +# +# Just kidding, image-checksum.sh is only intended to regression-test +# the ElectroBSD build system. +# +# Also note that this script relies on non-standardized output of other +# tools which might occasionally change. To be able to reproduce partial +# image checksums you thus need a userland that is close enough to the +# one that was used to create the original version. +# +########################################################################## + +UFS_PARTITION=p2 +EXPECTED_PARTITIONS=2 +MOUNTPOINT=/mnt +VERBOSE=0 +MTREE_KEYWORDS=size,time,uid,gid,sha256 + +verbose_log() { + local message="$*" + if [ "${VERBOSE}" = 0 ]; then + return + fi + echo "${message}" +} + +create_mtree_spec_file() { + local md_unit spec_file + + md_unit="${1}" + spec_file="${2}" + + verbose_log "Mounting /dev/md${md_unit}${UFS_PARTITION} at ${MOUNTPOINT}" + mount -o ro "/dev/md${md_unit}${UFS_PARTITION}" "${MOUNTPOINT}" || return 1 + + verbose_log "Running mtree, saving spec in ${spec_file}" + mtree -c -k "${MTREE_KEYWORDS}" -p "${MOUNTPOINT}" | mtree -C -k all > "${spec_file}" || return 1 + + verbose_log "Unmounting ${MOUNTPOINT} ..." + umount "${MOUNTPOINT}" || return 1 +} + +partition_count_acceptable() { + local md_unit="${1}" + + # Verify that there are exactly two partitions present + partitions=$(gpart show -r -p "md${md_unit}" | grep -c "md${md_unit}"p) + if [ "${partitions}" != "${EXPECTED_PARTITIONS}" ]; then + echo "Invalid number of partitions: ${partitions}" + return 1; + fi +} + +main() { + local image_file \ + args md_unit spec_file gpart_file + + args=$(getopt v $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + image_file=${1} + if [ -z "${image_file}" ]; then + echo "No image file provided" + return 1 + fi + spec_file="${image_file}.mtree" + if [ -f "${spec_file}" ]; then + echo "Spec file ${spec_file} already exists" + return 1 + fi + gpart_file="${image_file}.gpart" + if [ -f "${spec_file}" ]; then + echo "gpart file ${gpart_file} already exists" + return 1 + fi + + md_unit=$(mdconfig -o readonly -n -f "${image_file}") + if [ $? != 0 ]; then + return 1 + fi + + partition_count_acceptable "${md_unit}" || return 1 + + if [ ! -f "${spec_file}" ]; then + create_mtree_spec_file "${md_unit}" "${spec_file}" || return 1 + fi + if [ ! -f "${gpart_file}" ]; then + gpart list "md${md_unit}" | sed -E -e "s@(: md)${md_unit}@\1X@" > "${gpart_file}" + fi + + gpart_checksum=$(sha256 -q "${gpart_file}") + verbose_log "gpart checksum: ${gpart_checksum}" + + mdconfig -d -u "${md_unit}" || return 1 + + bootcode_checksum=$(dd if=/dev/md${md_unit}p1 2>/dev/null | sha256) + verbose_log "Boot code checksum: ${bootcode_checksum}" + mtree_checksum=$(sha256 -q "${spec_file}") + verbose_log "mtree checksum: ${mtree_checksum}" + + weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) + echo "Partial image checksum: ${weak_image_checksum}" + +} + +main "${@}" -- 2.7.0 From 66bdeef5f1e2b6ddde5c7038b215ac79348ad699 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 19:54:10 +0200 Subject: [PATCH 094/213] image-checksum: Add -r flag to reuse cache files --- release/scripts/image-checksum.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 6f79a81..1940921 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -50,6 +50,7 @@ EXPECTED_PARTITIONS=2 MOUNTPOINT=/mnt VERBOSE=0 MTREE_KEYWORDS=size,time,uid,gid,sha256 +REUSE_EXISTING_CACHE_FILES=false verbose_log() { local message="$*" @@ -90,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt v $*) + args=$(getopt rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -98,6 +99,10 @@ main() { set -- ${args} while true; do case "$1" in + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; -v) VERBOSE=1 shift @@ -116,12 +121,12 @@ main() { spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi gpart_file="${image_file}.gpart" if [ -f "${spec_file}" ]; then echo "gpart file ${gpart_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi md_unit=$(mdconfig -o readonly -n -f "${image_file}") -- 2.7.0 From d5666d81200183ba5860c058c2b67f5548f5db28 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 20:20:01 +0200 Subject: [PATCH 095/213] image-checksum.sh: Allow to overwrite the mtree flags --- release/scripts/image-checksum.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 1940921..cc31e90 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -91,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt rv $*) + args=$(getopt m:rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -99,6 +99,11 @@ main() { set -- ${args} while true; do case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; -r) REUSE_EXISTING_CACHE_FILES=true shift -- 2.7.0 From abd3d274b7ffabbfb13ec9f6014bd36455ee95d5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 31 Aug 2015 10:51:16 +0200 Subject: [PATCH 096/213] release/scripts/image-checksum.sh: Allow to checksum multiple images at once --- release/scripts/image-checksum.sh | 79 ++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index cc31e90..30ac62a 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -87,42 +87,11 @@ partition_count_acceptable() { fi } -main() { +generate_partial_image_checksum() { local image_file \ - args md_unit spec_file gpart_file - - args=$(getopt m:rv $*) - if [ $? -ne 0 ]; then - echo 'You are doing it wrong: Invalid flag specified' - exit 2 - fi - set -- ${args} - while true; do - case "$1" in - -m) - shift - MTREE_KEYWORDS="${1}" - shift - ;; - -r) - REUSE_EXISTING_CACHE_FILES=true - shift - ;; - -v) - VERBOSE=1 - shift - ;; - --) - shift; break - ;; - esac - done + md_unit spec_file gpart_file - image_file=${1} - if [ -z "${image_file}" ]; then - echo "No image file provided" - return 1 - fi + image_file="${1}" spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" @@ -159,8 +128,48 @@ main() { verbose_log "mtree checksum: ${mtree_checksum}" weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) - echo "Partial image checksum: ${weak_image_checksum}" + echo "Partial image checksum for ${image_file}: ${weak_image_checksum}" +} +main() { + local image_file \ + args + + args=$(getopt m:rv $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + if [ -z "${1}" ]; then + echo "No image file provided" + return 1 + fi + + for image_file in "${@}"; do + generate_partial_image_checksum "${image_file}" || return 1 + done } main "${@}" -- 2.7.0 From 2e307f76a5efabf743328fb28aec1c327fd337b4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:38:48 +0200 Subject: [PATCH 097/213] sys: Do not embed the compiler version in the kernel binary ... as it makes reproducing the binary with a different compiler more complicated. In case of ElectroBSD the compiler used can be usually deduced from the uname output anyway as the upstream revision is part of the fake hostname when using reproduce.sh. --- sys/conf/newvers.sh | 2 -- sys/kern/init_main.c | 1 - sys/kern/kern_mib.c | 3 --- sys/sys/systm.h | 1 - 4 files changed, 7 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 8717307..144bd33 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -115,7 +115,6 @@ else t=`date` fi i=`${MAKE:-make} -V KERN_IDENT` -compiler_v=$($(${MAKE:-make} -V CC) -v 2>&1 | grep -w 'version') for dir in /usr/bin /usr/local/bin; do if [ ! -z "${svnversion}" ] ; then @@ -240,7 +239,6 @@ $COPYRIGHT char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR; char version[sizeof(VERSTR) > 256 ? sizeof(VERSTR) : 256] = VERSTR; -char compiler_version[] = "${compiler_v}"; char ostype[] = "${TYPE}"; char osrelease[sizeof(RELSTR) > 32 ? sizeof(RELSTR) : 32] = RELSTR; int osreldate = ${RELDATE}; diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 8d5580b..f4bd65d 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -341,7 +341,6 @@ print_version(void *data __unused) while (len > 0 && version[len - 1] == '\n') len--; printf("%.*s %s\n", len, version, machine); - printf("%s\n", compiler_version); } SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t, diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c index 2ee260d..30f8d73 100644 --- a/sys/kern/kern_mib.c +++ b/sys/kern/kern_mib.c @@ -97,9 +97,6 @@ SYSCTL_INT(_kern, KERN_OSREV, osrevision, CTLFLAG_RD|CTLFLAG_CAPRD, SYSCTL_STRING(_kern, KERN_VERSION, version, CTLFLAG_RD|CTLFLAG_MPSAFE, version, 0, "Kernel version"); -SYSCTL_STRING(_kern, OID_AUTO, compiler_version, CTLFLAG_RD|CTLFLAG_MPSAFE, - compiler_version, 0, "Version of compiler used to compile kernel"); - SYSCTL_STRING(_kern, KERN_OSTYPE, ostype, CTLFLAG_RD|CTLFLAG_MPSAFE| CTLFLAG_CAPRD, ostype, 0, "Operating system type"); diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 026a03c..7cd7abc 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -50,7 +50,6 @@ extern int suspend_blocked; /* block suspend due to pending shutdown */ extern int rebooting; /* kern_reboot() has been called. */ extern const char *panicstr; /* panic message */ extern char version[]; /* system version */ -extern char compiler_version[]; /* compiler version */ extern char copyright[]; /* system copyright */ extern int kstack_pages; /* number of kernel stack pages */ -- 2.7.0 From 925699f0ff9cf622e7e7355990bd12c9a230a647 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:42:43 +0200 Subject: [PATCH 098/213] sys/conf/newvers.sh: Ditch support for p4 (non-free) and hg (not relevant for ElectroBSD) --- sys/conf/newvers.sh | 46 ++-------------------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 144bd33..3cf1023 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -141,11 +141,6 @@ if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then fi fi -for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/p4" ] && [ -z ${p4_cmd} ] ; then - p4_cmd=${dir}/p4 - fi -done if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -155,15 +150,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -d "${SYSDIR}/../.hg" ] ; then - for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/hg" ] ; then - hg_cmd="${dir}/hg -R ${SYSDIR}/.." - break - fi - done -fi - if [ -n "$svnversion" ] ; then svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` case "$svn" in @@ -203,38 +189,10 @@ if [ -n "$git_cmd" ] ; then # fi fi -if [ -n "$p4_cmd" ] ; then - p4version=`cd ${SYSDIR} && $p4_cmd changes -m1 "./...#have" 2>&1 | \ - awk '{ print $2 }'` - case "$p4version" in - [0-9]*) - p4version=" ${p4version}" - p4opened=`cd ${SYSDIR} && $p4_cmd opened ./... 2>&1` - case "$p4opened" in - File*) ;; - //*) p4version="${p4version}+edit" ;; - esac - ;; - *) unset p4version ;; - esac -fi - -if [ -n "$hg_cmd" ] ; then - hg=`$hg_cmd id 2>/dev/null` - svn=`$hg_cmd svn info 2>/dev/null | \ - awk -F': ' '/Revision/ { print $2 }'` - if [ -n "$svn" ] ; then - svn=" r${svn}" - fi - if [ -n "$hg" ] ; then - hg=" ${hg}" - fi -fi - cat << EOF > vers.c $COPYRIGHT -#define SCCSSTR "@(#)${VERSION} #${v}${svn}${git}${hg}${p4version}: ${t}" -#define VERSTR "${VERSION} #${v}${svn}${git}${hg}${p4version}: ${t}\\n ${u}@${h}:${d}\\n" +#define SCCSSTR "@(#)${VERSION} #${v}${svn}${git}: ${t}" +#define VERSTR "${VERSION} #${v}${svn}${git}: ${t}\\n ${u}@${h}:${d}\\n" #define RELSTR "${RELEASE}" char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR; -- 2.7.0 From c10fa4c2b89695d011c0bfce765d5d847175d5ca Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 1 Jun 2015 12:46:24 +0200 Subject: [PATCH 099/213] contrib/mdocml: Scan files in deterministical order ... to actually get reproducible mandoc.db files. --- contrib/mdocml/mandocdb.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/contrib/mdocml/mandocdb.c b/contrib/mdocml/mandocdb.c index 08f89c1..3cb032b 100644 --- a/contrib/mdocml/mandocdb.c +++ b/contrib/mdocml/mandocdb.c @@ -563,6 +563,19 @@ usage: } /* + * find_compare (copied from usr.bin/find/find.c) -- + * tell fts_open() how to order the traversal of the hierarchy. + * This variant gives lexicographical order, i.e., alphabetical + * order within each directory. + */ +static int +find_compare(const FTSENT * const *s1, const FTSENT * const *s2) +{ + + return (strcoll((*s1)->fts_name, (*s2)->fts_name)); +} + +/* * Scan a directory tree rooted at "basedir" for manpages. * We use fts(), scanning directory parts along the way for clues to our * section and architecture. @@ -592,7 +605,7 @@ treescan(void) argv[1] = (char *)NULL; f = fts_open((char * const *)argv, - FTS_PHYSICAL | FTS_NOCHDIR, NULL); + FTS_PHYSICAL | FTS_NOCHDIR, find_compare); if (f == NULL) { exitcode = (int)MANDOCLEVEL_SYSERR; say("", "&fts_open"); -- 2.7.0 From b142bd5b76c15363606cf067eb54dd12a4b1498d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 3 Jun 2015 17:16:10 +0200 Subject: [PATCH 100/213] mandocdb: Normalize inodevs to get reproducible results This is a proof-of-concept that scales poorly due to linear searches. The proper fix is probably to use hash-based lookups and only do it when the user wants reproducible results. I didn't find any automated tests for this so the only testing done is "does 'man' appear to be usable". It indeed appears to be ... --- contrib/mdocml/mandocdb.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/contrib/mdocml/mandocdb.c b/contrib/mdocml/mandocdb.c index 3cb032b..5da109e 100644 --- a/contrib/mdocml/mandocdb.c +++ b/contrib/mdocml/mandocdb.c @@ -933,6 +933,34 @@ filescan(const char *file) mlink_add(mlink, &st); } +/* + * Messes up inodevs in a reproducible way as long as + * the call order does not change. The implementation + * is silly and only used as proof of concept. + */ +#define HOPEFULLY_ENOUGH_FOR_EVERYBODY 5000 +static void +normalize_inodev(struct inodev *inodev) { + static size_t table[HOPEFULLY_ENOUGH_FOR_EVERYBODY]; + size_t key; + int i; + + key = inodev->st_ino + inodev->st_dev; + + for (i = 0; i < sizeof(table)/sizeof(table[0]); i++) { + if (table[i] == 0) { + /* New value, add to table*/ + table[i] = key; + } + if (table[i] == key) { + /* Use index as new value */ + inodev->st_ino = i; + inodev->st_dev = i; + return; + } + } +} + static void mlink_add(struct mlink *mlink, const struct stat *st) { @@ -963,6 +991,7 @@ mlink_add(struct mlink *mlink, const struct stat *st) memset(&inodev, 0, sizeof(inodev)); /* Clear padding. */ inodev.st_ino = st->st_ino; inodev.st_dev = st->st_dev; + normalize_inodev(&inodev); slot = ohash_lookup_memory(&mpages, (char *)&inodev, sizeof(struct inodev), inodev.st_ino); mpage = ohash_find(&mpages, slot); -- 2.7.0 From 44e3c6345e2eaf3b183222e814001e55d13d3ee1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 1 Jun 2015 13:56:05 +0200 Subject: [PATCH 101/213] release/Makefile: Build the src.txz only once and also fake source ownership ... after hours of blood, sweat and tears it finally (appears to) work. Keep release/scripts/tar-time-reset.sh around but update a comment to make it obvious that the script isn't used anymore --- release/Makefile | 27 +++++++++++++++++++-------- release/scripts/tar-time-reset.sh | 8 +++----- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/release/Makefile b/release/Makefile index 4c2b530..b968a02 100644 --- a/release/Makefile +++ b/release/Makefile @@ -150,11 +150,25 @@ kernel.txz: src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src - cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist \ - --exclude "usr/src/release/*.txz" \ - --exclude "usr/src/release/*.orig" usr/src | \ - ${XZ_CMD} > ${.OBJDIR}/src.txz +# Create an mtree spec with faked timestamps so we get a reproducible +# tar file. We do not use tar for this because it's mtree generator +# appears to be buggy and exits with an memory allocation failure. +# +# It's important that the excluded paths start with "./", otherwise +# file locations are not recorded correctly, and, for example, +# usr/src/usr.sbin appears as usr/src/release/usr.sbin in the +# tar file. Only the shadow knows if that's a bug or a feature. + echo "./usr/src/.git" >${.OBJDIR}/mtree-exclude + echo "./usr/src/release/dist" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/src.mtree" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/mtree-exclude" >>${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && mtree -c -L -k time -X ${.OBJDIR}/mtree-exclude | \ + sed -E -e 's@(time=)[0-9]+\.[0-9]+@\1${EPOCH_DATE}.0 uid=0 gid=0@' \ + > ${.OBJDIR}/src.mtree + rm ${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && tar cLvf - @${.OBJDIR}/src.mtree \ + | ${XZ_CMD} > ${.OBJDIR}/src.txz + rm ${.OBJDIR}/src.mtree ports.txz: mkdir -p ${DISTDIR}/usr @@ -271,9 +285,6 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in ${EXTRA_PACKAGES}; do \ - sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ - done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh index 8186fd8..f1b8434 100755 --- a/release/scripts/tar-time-reset.sh +++ b/release/scripts/tar-time-reset.sh @@ -17,11 +17,9 @@ ########################################################################## # # This script resets the timestamps in a given tarfile to hopefully make -# it reproducible. This is a rather wasteful approach, but works for now. -# -# A better solution would be to patch bsdtar to optionally use a fixed -# time (without having to clown around with mtree specs). -# +# it reproducible. As this is a rather wasteful approach the script is +# no longer used. It hasn't been removed yet as it may be useful for +# testing purposes. ########################################################################## main() { -- 2.7.0 From 7d028f8e60240473b6c770abefc12ffcb62c8d66 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Jun 2015 12:18:16 +0200 Subject: [PATCH 102/213] release/Makefile: Reorder dependencies to reduce the chances that base or kernel parts end up in the src.txz. This is merely a workaround --- release/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index b968a02..a84fc5e 100644 --- a/release/Makefile +++ b/release/Makefile @@ -284,7 +284,7 @@ mini-memstick: mini-memstick.img mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} -packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} +packagesystem: ${EXTRA_PACKAGES} base.txz kernel.txz sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} -- 2.7.0 From fbe9963b2269441e32f71a0b29291fc205b4ffa2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 17:30:25 +0200 Subject: [PATCH 103/213] Add strip-freebsd.sh ... which suggests a bunch of stuff to delete from a vanilla FreeBSD checkout. In a previous life it was called free-freebsd.sh which obviously sounds more awesome, but nowadays most of the suggested stuff for removal is actually free software that just isn't relevant for ElectroBSD. Removing code we don't need means we don't have to care about its security and license issues. There's lots of code to remove left! --- release/scripts/strip-freebsd.sh | 171 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100755 release/scripts/strip-freebsd.sh diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh new file mode 100755 index 0000000..eaaf378 --- /dev/null +++ b/release/scripts/strip-freebsd.sh @@ -0,0 +1,171 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# This script pretends to free a FreeBSD checkout from +# known-unfree files and other stuff that is not required +# by ElectroBSD. +# +# While it is pretty much guaranteed to ditch a bunch of files +# it doesn't work very thorougly and the resulting checkout is +# likely to still contain lots of non-free parts that haven't +# been discovered yet. + +get_snd_csa_files() { + find sys/dev/sound/pci -name "csa*" +} + +get_snd_ds1_files() { + find sys/dev/sound/pci -name "ds1*" +} + +get_snd_maestro3_files() { + # This file contains the actual blobs + echo sys/dev/sound/pci/allegro_code.h + + # This file contains the (free) code that relies on the blobs. + echo sys/dev/sound/pci/maestro3.c +} + +# We can't simply remove the whole directory +# as a (free) header is required by bge. +get_bce_files() { + echo "sys/dev/bce/if_bcefw.h" \ + "sys/dev/bce/if_bce.c" +} + +get_usb_firmware_files() { + find sys/dev/usb/ -name "*fw*" +} + +# XXX: Misleading name, some of the files are merely tainted +# by non-free dependencies +get_unfree_files() { + find sys/ -name "*.uu" + get_snd_csa_files + get_snd_ds1_files + get_snd_maestro3_files + get_bce_files + get_usb_firmware_files +} + +get_files_to_ditch() { + get_unfree_files +} + +# These architectures are mainly unsupported by ElectroBSD +# due to lack of hardware for testing purposes. +# +# The source directories are mainly removed to shrink the +# source tarball and to reduce the number of files that +# should be audited for license and security issues. +get_unsupported_architectures() { + echo "arm arm64 mips pc98 powerpc sparc64" +} + +# These depend on or contain proprietary firmware that is included in sys/contrib/dev +get_tainted_sys_contrib_devs() { + echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu run uath urtwn wpi" +} + +# These require proprietary firmware that is included in sys/dev +# and may cause build failures without it. +get_tainted_sys_devs() { + # bce has already been taken care of by get_bce_files() above + echo "bxe ctau cx cxgb cxgbe ispfw qlxgbe" \ + "spibus it tw" +} + +get_unused_contrib_dirs() { + # XXX: gcc can't be deleted because parts of it are apparently + # required to build libc. This should be investigated more thoroughly, + # hopefully it can be fixed. + echo "apr apr-util ipfilter ofed sendmail serf subversion tcsh" +} + +get_directories_to_ditch() { + local arch \ + dir arch_dir sys_contrib contrib_dir + + for dir in sys sys/boot; do + for arch in $(get_unsupported_architectures); do + potential_directory="${dir}/${arch}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done + done + + for sys_contrib in ipfilter octeon-sdk; do + echo "sys/contrib/${sys_contrib}" + done + + for sys_contrib in $(get_tainted_sys_contrib_devs); do + echo "sys/contrib/dev/${sys_contrib}" + done + + for sys_dev in $(get_tainted_sys_devs); do + echo "sys/dev/${sys_dev}" + done + + for contrib_dir in $(get_unused_contrib_dirs); do + potential_directory="contrib/${contrib_dir}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done +} + +purify_cwd() { + # There are no spaces in paths or file names. + files_to_ditch="$(get_files_to_ditch)" + for f in $files_to_ditch; do + [ -f "${f}" ] && echo "rm ${f}" + done + + dirs_to_ditch="$(get_directories_to_ditch)" + for d in $dirs_to_ditch; do + [ -d "${d}" ] && echo "rm -r ${d}" + done +} + +main() { + local src_dir \ + files_to_ditch dirs_to_ditch + + src_dir="${1}" + if [ -z "${src_dir}" ]; then + echo "No source directory given" + return 1 + fi + if [ ! -d "${src_dir}" ]; then + echo "No such directory: ${src_dir}" + return 1 + fi + # Make it less likely to operate on a directory + # that isn't actually a FreeBSD checkout + if [ ! -f "${src_dir}/COPYRIGHT" ]; then + echo "${src_dir} contains no COPYRIGHT file" + return 1 + fi + + cd "${src_dir}" || return 1 + + purify_cwd +} + +main "${@}" -- 2.7.0 From 0b31e186b9755a69da5dd95f8a417a76248c228c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 1 Jan 2016 11:22:49 +0100 Subject: [PATCH 104/213] release/scripts/strip-freebsd.sh: Ditch rtwn which got added in r293009 --- release/scripts/strip-freebsd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index eaaf378..56be187 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -79,7 +79,7 @@ get_unsupported_architectures() { # These depend on or contain proprietary firmware that is included in sys/contrib/dev get_tainted_sys_contrib_devs() { - echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu run uath urtwn wpi" + echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu rtwn run uath urtwn wpi" } # These require proprietary firmware that is included in sys/dev -- 2.7.0 From 47b0f4884c66dfa205241072a8cb5a25fac679de Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 23 Sep 2015 13:53:34 +0200 Subject: [PATCH 105/213] Makefile.inc1: Mark two suspicious mtree spec modifications as such --- Makefile.inc1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index ae291ed..4f40ea0 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1300,6 +1300,7 @@ packagekernel: .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) .if defined(EPOCH_DATE) +# XXX: Is this really necessary given that we already modify the METALOG itself? sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.meta .endif @@ -1314,6 +1315,7 @@ packagekernel: ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz .if ${BUILDKERNELS:[#]} > 1 && !defined(NO_INSTALLEXTRAKERNELS) .for _kernel in ${BUILDKERNELS:[2..-1]} +# XXX: See XXX above .if defined(EPOCH_DATE) sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta -- 2.7.0 From d41850110f0943177b0b04472405495c6e44ec44 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 23 Sep 2015 13:56:57 +0200 Subject: [PATCH 106/213] Makefile.inc: Try to make the release reproducible when not running as root --- Makefile.inc1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 4f40ea0..f864ab2 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1014,7 +1014,8 @@ distributeworld installworld: _installcheck_world @# remove duplicated slashes which can occur if DISTDIR is undefined. @# Keeping them results in missing files in the distribution tarballs. .if defined(EPOCH_DATE) - sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + sed -E -e 's@time=[0-9]+.0@@' \ + -e 's@(type=)@uid=0 gid=0 time=${EPOCH_DATE}.0 \1@' \ -e 's@//@/@' -i '.bak' ${METALOG} .endif .for dist in base ${EXTRA_DISTRIBUTIONS} -- 2.7.0 From c4a2eab4df365b51d09daf478c67bdde6ad77955 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:33:31 +0200 Subject: [PATCH 107/213] sys/conf/newvers.sh: Remove svn support (but keep 'git svn' support) --- sys/conf/newvers.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 3cf1023..631dfcb 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -116,31 +116,6 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -for dir in /usr/bin /usr/local/bin; do - if [ ! -z "${svnversion}" ] ; then - break - fi - if [ -x "${dir}/svnversion" ] && [ -z ${svnversion} ] ; then - # Run svnversion from ${dir} on this script; if return code - # is not zero, the checkout might not be compatible with the - # svnversion being used. - ${dir}/svnversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=${dir}/svnversion - break - fi - fi -done - -if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then - /usr/bin/svnliteversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=/usr/bin/svnliteversion - else - svnversion= - fi -fi - if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -150,14 +125,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -n "$svnversion" ] ; then - svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` - case "$svn" in - [0-9]*) svn=" r${svn}" ;; - *) unset svn ;; - esac -fi - if [ -n "$git_cmd" ] ; then git=`$git_cmd rev-parse --verify --short HEAD 2>/dev/null` svn=`$git_cmd svn find-rev $git 2>/dev/null` -- 2.7.0 From 65235ddb87a419cb7ee059882247645a56e5f75e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:34:45 +0200 Subject: [PATCH 108/213] sys/conf/newvers.sh: Do not add git hash from .git if KERNEL_VERSION_NUMBER is defined ... as this indicates that we are (trying to) reproduce a build. Do not use REPRO_SEED as it's always set nowadays. XXX: There should be a cleaner way to do this. --- sys/conf/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 631dfcb..9a99751 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -116,7 +116,7 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -if [ -d "${SYSDIR}/../.git" ] ; then +if [ -z "${KERNEL_VERSION_NUMBER}" -a -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then git_cmd="${dir}/git --git-dir=${SYSDIR}/../.git" -- 2.7.0 From 246fdc27d0c8ccc69d0868c01822c3efde8db38f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 28 May 2015 15:04:48 +0200 Subject: [PATCH 109/213] sys/conf/newvers.sh: Allow to overwrite the kernel version ... as the object directory may be dirty. --- sys/conf/newvers.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 9a99751..4894a51 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -97,7 +97,7 @@ then fi touch version -v=`cat version` +v=${KERNEL_VERSION_NUMBER:-`cat version`} u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} @@ -170,4 +170,6 @@ int osreldate = ${RELDATE}; char kern_ident[] = "${i}"; EOF -echo $((v + 1)) > version +if [ -z "${KERNEL_VERSION_NUMBER}" ]; then + echo $((v + 1)) > version +fi -- 2.7.0 From 0cad3fa6f9e264d7e3b05d485c50586a85c93516 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Sep 2015 11:53:45 +0200 Subject: [PATCH 110/213] make-memstick.sh: Additionally fake uid and gid on the created fs --- release/amd64/make-memstick.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index a5d88e1..25c00a5 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -30,7 +30,12 @@ fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +# Prepare mtree spec to fake timestamp, owner and group. +# As a result, man pages will be owned by root instead of man. +# Unfortunately we can't simply reuse ${1}/METALOG as it is incomplete. +mtree -c -k time -p "${1}" | sed \ + -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ + > "${2}.mtree" || return 1 makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" -- 2.7.0 From 18ac29f3950dbb6699437ac136cca9cd3c3ff83e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 30 Sep 2015 13:15:44 +0200 Subject: [PATCH 111/213] release/amd64/make-memstick.sh: Error out if mkimg fails instead of cleaning up Makes debugging more convenient. --- release/amd64/make-memstick.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 25c00a5..a39a200 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -46,5 +46,9 @@ rm ${1}/etc/rc.conf.local rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +if [ $? -ne 0 ]; then + echo "mkimg failed" + exit 1 +fi rm ${2}.part -- 2.7.0 From c9175a5914c17896a1771e75c749bae82f40ff64 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:43:06 +0100 Subject: [PATCH 112/213] release/amd64/make-memstick.sh: Stop creating a puny swap partion --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index a39a200..226846f 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -45,7 +45,7 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.7.0 From b6ddde11f2e369c4b65a3077622360cea53b350d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:44:10 +0100 Subject: [PATCH 113/213] release/amd64/make-memstick.sh: Stop creating an EFI partition as the EFI goo does not build reproducible --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 226846f..3feded5 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -45,7 +45,7 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.7.0 From d073b3c4ee40ca18ea94653d350641101ce5869e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 17:26:59 +0200 Subject: [PATCH 114/213] Add reproduce.sh which makes reproducing ElectroBSD more convenient Squashed commits worth mentioning: - Add -j option to overwrite the maximum number make jobs - Assert that the source directory is untainted (according to strip-freebsd.sh) ... and add -a flag to remove offending files - Allow to resume a build by using the -r flag. --- reproduce.sh | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100755 reproduce.sh diff --git a/reproduce.sh b/reproduce.sh new file mode 100755 index 0000000..dac0e34 --- /dev/null +++ b/reproduce.sh @@ -0,0 +1,196 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# reproduce.sh +# +# Script to make reproducing an ElectroBSD build more convenient. +# Before using it, make sure BUILD and EPOCH contain values other +# than __BUILD__ and __EPOCH__, either by editing the script or +# by putting them in a configuration file that is speficied with +# the -f option. + +# These variables have to be set to the values used for the build +# that is supposed to be reproduced. ${SRC_DIR} must contain the +# matching sources! +BUILD=__BUILD__ +EPOCH=__EPOCH__ + +# This is just a suggestion, feel free to overwrite it with the -j option. +MAX_MAKE_JOBS="${MAX_MAKE_JOBS-4}" + +# Currently hardcoded. +SRC_DIR=/usr/src +OPTIONAL_CONFIG_FILE="${SRC_DIR}/reproduce.conf" + +# Make sure we respawn with the same script, even if it is located +# outside the SRC_DIR and called with a relative path. +REPRODUCE_SH="$(realpath "${0}")" + +# When set to true, existing object files will be reused. +# If the source files changed, the result will not be reproducible! +RESUME_BUILD="${RESUME_BUILD-false}" + +announce_status() { + local msg \ + timestamp + + msg="${*}" + timestamp=$(date "+%Y-%m-%d_%H:%M") + + echo "${timestamp}: ${msg}" +} + +reproduce_all_the_things() { + + if "${RESUME_BUILD}"; then + announce_status "Resuming ..." + export KERNFAST=1 + export NO_CLEAN=1 + fi + + announce_status "Starting to build the kernel" + make buildkernel || return 1 + + announce_status "Starting to build the world" + make -j${MAX_MAKE_JOBS} buildworld || return 1 + + # Make sure obj files aren't dumped in ${SRC_DIR} + mkdir -p "/usr/obj${SRC_DIR}/release" || return 1 + + if ! "${RESUME_BUILD}"; then + announce_status "Starting to clean the release dir" + make -C "${SRC_DIR}/release" clean + fi + announce_status "Starting to build the release" + time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 + + announce_status "Done with release memstick for ${REPRO_SEED}" +} + +assert_untainted_source_tree() { + local auto_untaint \ + untaint_commands + + auto_untaint="${1}" + + untaint_commands="$(sh ./release/scripts/strip-freebsd.sh .)" + if [ -n "${untaint_commands}" ] ; then + if $auto_untaint; then + echo "Auto untainting $(pwd)" + echo "${untaint_commands}" | sh -x || return 1 + else + echo "${SRC_DIR} is tainted. Use -a flag to auto-untaint it." + return 1 + fi + fi + return 0 +} + +respawn_with_clean_environment() { + exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ + LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ + MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ + REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" +} + +main() { + local args \ + auto_untaint config_file fake_user dry_run + + fake_user=elektropunker + + auto_untaint=false + dry_run=false + config_file="${OPTIONAL_CONFIG_FILE}" + + args=$(getopt af:j:nr $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -a) + shift + auto_untaint="true" + ;; + -j) + shift; + MAX_MAKE_JOBS="${1}" + shift; + ;; + -f) + shift; + config_file="${1}" + shift; + if [ ! -f "${config_file}" ]; then + echo "Config file ${config_file} does not exist" + exit 2 + fi + ;; + -n) + dry_run=true + shift + ;; + -r) + shift + RESUME_BUILD=true + ;; + --) + shift; break + ;; + esac + done + + if [ -f "${config_file}" ]; then + announce_status "Reading config from ${config_file}" + . "${config_file}" || exit 2 + fi + + if [ -n "${ALREADY_RESPAWNED}" -a "${ALREADY_RESPAWNED}" = 1 ]; then + if [ -z "${REPRO_SEED}" ]; then + announce_status "Respawned with REPRO_SEED unset" + return 1 + fi + reproduce_all_the_things + return + fi + if [ "${BUILD}" = "__BUILD__" ]; then + announce_status "BUILD not set" + return 1 + fi + if [ "${EPOCH}" = "__EPOCH__" ]; then + announce_status "EPOCH not set" + return 1 + fi + + export REPRO_SEED="${fake_user}:${BUILD}:${EPOCH}" + + announce_status "REPRO_SEED=${REPRO_SEED}" + + cd "${SRC_DIR}" || return 1 + + assert_untainted_source_tree "${auto_untaint}" || return 1 + + if ! $dry_run; then + respawn_with_clean_environment + fi +} + +main "${@}" -- 2.7.0 From 32e43ce395519cc84cac692be51f8f455a446619 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Nov 2015 18:55:18 +0100 Subject: [PATCH 115/213] share/doc: Detach 'legal' from the build It (tries to) install proprietary licenses for code that is not part of ElectroBSD and deleted by reproduce.sh's auto-untaint mode (-a). --- share/doc/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/share/doc/Makefile b/share/doc/Makefile index 7a02b29..5b5e507 100644 --- a/share/doc/Makefile +++ b/share/doc/Makefile @@ -5,7 +5,6 @@ SUBDIR= ${_IPv6} \ ${_atf} \ - legal \ ${_llvm} \ ${_pjdfstest} \ ${_roffdocs} -- 2.7.0 From 1be19dfb3a4e6731e8f886e6d8118ed4bd948e2c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 13:38:21 +0200 Subject: [PATCH 116/213] sys/boot/Makefile.amd64: Detach efi from the built It doesn't built reproducible, among other things due to the embedded file system, and none of the ElectroBSD users I'm aware of have efi-capable systems anyway. --- sys/boot/Makefile.amd64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/Makefile.amd64 b/sys/boot/Makefile.amd64 index 384cf7a..586736d 100644 --- a/sys/boot/Makefile.amd64 +++ b/sys/boot/Makefile.amd64 @@ -1,6 +1,6 @@ # $FreeBSD$ -SUBDIR+= efi +#SUBDIR+= efi SUBDIR+= libstand32 SUBDIR+= zfs SUBDIR+= userboot -- 2.7.0 From 5526d09acf70ec35443bbf7cdaa34141e69de9a8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 May 2015 18:40:56 +0200 Subject: [PATCH 117/213] release/amd64/make-memstick.sh: Allow to overwrite VOLUME_LABEL through the environment --- release/amd64/make-memstick.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 3feded5..66ee073 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -12,6 +12,7 @@ PATH=/bin:/usr/bin:/sbin:/usr/sbin export PATH +VOLUME_LABEL=${VOLUME_LABEL-"FreeBSD_Install"} if [ $# -ne 2 ]; then echo "make-memstick.sh /path/to/directory /path/to/image/file" @@ -28,7 +29,7 @@ if [ -e ${2} ]; then exit 1 fi -echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab +echo "/dev/ufs/${VOLUME_LABEL} / ufs ro,noatime 1 1" > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local # Prepare mtree spec to fake timestamp, owner and group. # As a result, man pages will be owned by root instead of man. @@ -36,7 +37,7 @@ echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local mtree -c -k time -p "${1}" | sed \ -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ > "${2}.mtree" || return 1 -makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} +makefs -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 -- 2.7.0 From 35b5c25c09fa4db671b1b697545b6555b75a1b31 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 May 2015 18:42:42 +0200 Subject: [PATCH 118/213] release/Makefile: Export VOLUME_LABEL for make-memstick.sh Adding the UNAME_s to it probably would not hurt ... --- release/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index a84fc5e..76737e1 100644 --- a/release/Makefile +++ b/release/Makefile @@ -77,9 +77,11 @@ VOLUME_LABEL= ${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET_ARCH} .endif .if !defined(VOLUME_LABEL) || empty(VOLUME_LABEL) -VOLUME_LABEL= FreeBSD_Install +VOLUME_LABEL= FreeBSD_Install .endif +.export VOLUME_LABEL + .if !exists(${DOCDIR}) NODOC= true .endif -- 2.7.0 From c188d50eb8a336a7ca79c13af912d11981bfac01 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 3 Dec 2015 18:34:21 +0100 Subject: [PATCH 119/213] lib/libkvm: Unbreak the build without kernel sources that don't ship with ElectroBSD --- lib/libkvm/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/libkvm/Makefile b/lib/libkvm/Makefile index 33f8e5b..8e89f81 100644 --- a/lib/libkvm/Makefile +++ b/lib/libkvm/Makefile @@ -11,13 +11,8 @@ WARNS?= 3 SRCS= kvm.c kvm_cptime.c kvm_file.c kvm_getloadavg.c \ kvm_getswapinfo.c kvm_pcpu.c kvm_proc.c kvm_vnet.c \ - kvm_minidump_aarch64.c \ kvm_amd64.c kvm_minidump_amd64.c \ - kvm_arm.c kvm_minidump_arm.c \ - kvm_i386.c kvm_minidump_i386.c \ - kvm_minidump_mips.c \ - kvm_powerpc.c kvm_powerpc64.c \ - kvm_sparc64.c + kvm_i386.c kvm_minidump_i386.c INCS= kvm.h LIBADD= elf -- 2.7.0 From 735ab383915d06f44905e453a6ce8ca861c9f913 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 May 2015 14:03:06 +0200 Subject: [PATCH 120/213] usr.sbin/pkg: Change URL_SCHEME_PREFIX to an URL more easily controlled by the user ... through ssh port forwarding. --- usr.sbin/pkg/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.sbin/pkg/config.c b/usr.sbin/pkg/config.c index 2fea11f..8cc35190 100644 --- a/usr.sbin/pkg/config.c +++ b/usr.sbin/pkg/config.c @@ -64,7 +64,7 @@ static struct config_entry c[] = { [PACKAGESITE] = { PKG_CONFIG_STRING, "PACKAGESITE", - URL_SCHEME_PREFIX "http://pkg.FreeBSD.org/${ABI}/latest", + URL_SCHEME_PREFIX "http://127.0.0.1:8000/packages/${ABI}/", NULL, NULL, false, -- 2.7.0 From d6e037129411b75fabcef6308e96794abac311a9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 15:33:59 +0200 Subject: [PATCH 121/213] share/doc: Exclude directories that contain (partly) non-free materials --- share/doc/Makefile | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/share/doc/Makefile b/share/doc/Makefile index 5b5e507..dc2003f 100644 --- a/share/doc/Makefile +++ b/share/doc/Makefile @@ -3,11 +3,9 @@ .include -SUBDIR= ${_IPv6} \ - ${_atf} \ +SUBDIR= ${_atf} \ ${_llvm} \ - ${_pjdfstest} \ - ${_roffdocs} + ${_pjdfstest} .if ${MK_TESTS} != "no" _atf= atf @@ -18,14 +16,11 @@ _pjdfstest= pjdfstest _llvm= llvm .endif -.if ${MK_INET6} != "no" -_IPv6= IPv6 -.endif - -# FIXME this is not a real solution ... -.if ${MK_GROFF} != "no" -_roffdocs= papers psd smm usd -.endif +# Note: Documents in the sub directories IPv6, papers, psd, smm +# and usd are not installed on ElectroBSD due to license problems. +# +# Some of the excluded papers actually have free licenses and +# should eventually be installed. SUBDIR_PARALLEL= -- 2.7.0 From 6b046461a4ebe1f81c7ba72c94b0436b48abd894 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 10:10:43 +0200 Subject: [PATCH 122/213] geom: Do not build raid, raid3 and vinum classes --- sys/modules/geom/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/sys/modules/geom/Makefile b/sys/modules/geom/Makefile index f54a3e8..cb7939f 100644 --- a/sys/modules/geom/Makefile +++ b/sys/modules/geom/Makefile @@ -16,14 +16,11 @@ SUBDIR= geom_bde \ geom_multipath \ geom_nop \ geom_part \ - geom_raid \ - geom_raid3 \ geom_sched \ geom_shsec \ geom_stripe \ geom_uncompress \ geom_uzip \ - geom_vinum \ geom_virstor \ geom_zero -- 2.7.0 From 7fe2b10ed2d7f653fbe42012bd62a78bcb63cf41 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 12:17:36 +0200 Subject: [PATCH 123/213] share/mk/src.opts.mk: Disable a bunch of options by default .. either due to license issues or because they are not considered relevant for the majority of ElectroBSD users. XXX: This commit does not regenerate src.conf(5) as the process hasn't been reverse-engineered yet. --- share/mk/src.opts.mk | 58 ++++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 678c31e..4e0511b 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -49,30 +49,25 @@ __DEFAULT_YES_OPTIONS = \ AMD \ APM \ AT \ - ATM \ AUDIT \ AUTHPF \ AUTOFS \ BHYVE \ BINUTILS \ BINUTILS_BOOTSTRAP \ - BLUETOOTH \ BOOT \ BOOTPARAMD \ BOOTPD \ BSD_CPIO \ - BSDINSTALL \ BSNMP \ BZIP2 \ CALENDAR \ CAPSICUM \ CASPER \ - CCD \ CDDL \ CPP \ CROSS_COMPILER \ CRYPT \ - CTM \ CUSE \ CXX \ DICT \ @@ -85,14 +80,9 @@ __DEFAULT_YES_OPTIONS = \ FDT \ FILE \ FINGER \ - FLOPPY \ - FMTREE \ FORTH \ FP_LIBC \ - FREEBSD_UPDATE \ - FTP \ GAMES \ - GCOV \ GDB \ GNU \ GNU_GREP_COMPAT \ @@ -106,9 +96,7 @@ __DEFAULT_YES_OPTIONS = \ INET \ INET6 \ INETD \ - IPFILTER \ IPFW \ - ISCSI \ JAIL \ KDUMP \ KVM \ @@ -127,7 +115,6 @@ __DEFAULT_YES_OPTIONS = \ MAILWRAPPER \ MAKE \ MANDOCDB \ - NDIS \ NETCAT \ NETGRAPH \ NLS_CATALOGS \ @@ -135,35 +122,21 @@ __DEFAULT_YES_OPTIONS = \ NTP \ OPENSSL \ PAM \ - PC_SYSINSTALL \ PF \ PKGBOOTSTRAP \ - PMC \ - PORTSNAP \ - PPP \ QUOTAS \ RADIUS_SUPPORT \ - RCMDS \ - RBOOTD \ RCS \ RESCUE \ ROUTED \ - SENDMAIL \ SETUID_LOGIN \ SHAREDOCS \ - SOURCELESS \ - SOURCELESS_HOST \ - SOURCELESS_UCODE \ - SVNLITE \ SYSCONS \ TALK \ - TCP_WRAPPERS \ - TCSH \ TELNET \ TESTS \ TEXTPROC \ TFTP \ - TIMED \ UNBOUND \ USB \ UTMPX \ @@ -188,6 +161,37 @@ __DEFAULT_NO_OPTIONS = \ SORT_THREADS \ SVN +# Disable a bunch of additional options that default to yes in FreeBSD +__DEFAULT_NO_OPTIONS += \ + ATM \ + BLUETOOTH \ + BSDINSTALL \ + CCD \ + CTM \ + FLOPPY \ + FMTREE \ + FREEBSD_UPDATE \ + FTP \ + GCOV \ + IPFILTER \ + ISCSI \ + NDIS \ + RBOOTD \ + PC_SYSINSTALL \ + PMC \ + PORTSNAP \ + PPP \ + RCMDS \ + SENDMAIL \ + SVNLITE \ + SOURCELESS \ + SOURCELESS_HOST \ + SOURCELESS_UCODE \ + SYSINSTALL \ + TCP_WRAPPERS \ + TCSH \ + TIMED \ + # # Default behaviour of some options depends on the architecture. Unfortunately # this means that we have to test TARGET_ARCH (the buildworld case) as well -- 2.7.0 From 7c6730d38a1f2fecab55c1172e92b9081242c125 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 24 Jun 2015 12:53:20 +0200 Subject: [PATCH 124/213] share/mk/src.opts.mk: Enable CLANG_EXTRAS by default as it's required for llvm-symbolizer --- share/mk/src.opts.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 4e0511b..ada687f 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -149,7 +149,6 @@ __DEFAULT_YES_OPTIONS = \ __DEFAULT_NO_OPTIONS = \ BSD_GREP \ - CLANG_EXTRAS \ DTRACE_TESTS \ EISA \ HESIOD \ @@ -161,6 +160,10 @@ __DEFAULT_NO_OPTIONS = \ SORT_THREADS \ SVN +# Enable additional options that default to NO in FreeBSD +__DEFAULT_YES_OPTIONS += \ + CLANG_EXTRAS \ + # Disable a bunch of additional options that default to yes in FreeBSD __DEFAULT_NO_OPTIONS += \ ATM \ -- 2.7.0 From 8e6a3ffad81c0f65acc60340dd5ec0245b2b5a7c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 17:53:19 +0200 Subject: [PATCH 125/213] etc/master.passwd: Change root's login shell to sh Now that csh is no longer compiled by default that seems like a rather swell idea. --- etc/master.passwd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/master.passwd b/etc/master.passwd index 430fd2c..44a1609 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -1,6 +1,6 @@ # $FreeBSD$ # -root::0:0::0:0:Charlie &:/root:/bin/csh +root::0:0::0:0:Charlie &:/root:/bin/sh toor:*:0:0::0:0:Bourne-again Superuser:/root: daemon:*:1:1::0:0:Owner of many system processes:/root:/usr/sbin/nologin operator:*:2:5::0:0:System &:/:/usr/sbin/nologin -- 2.7.0 From c084ef8e78e7dd0d7464f44ae4b251c6f9ea38c2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 12:03:46 +0200 Subject: [PATCH 126/213] release/Makefile: Set German keyboard map --- release/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/release/Makefile b/release/Makefile index 76737e1..f7595fd 100644 --- a/release/Makefile +++ b/release/Makefile @@ -211,6 +211,7 @@ disc1: packagesystem ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf + echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf echo debug.witness.trace=0 >> ${.TARGET}/etc/sysctl.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf cp ${.CURDIR}/rc.local ${.TARGET}/etc -- 2.7.0 From 0dc08351964b273b8b546a11a2465fe3e0a6c21e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 12:06:14 +0200 Subject: [PATCH 127/213] release/Makefile: Set hostname to ${VOLUME_LABEL} --- release/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/release/Makefile b/release/Makefile index f7595fd..15d98dc 100644 --- a/release/Makefile +++ b/release/Makefile @@ -212,6 +212,7 @@ disc1: packagesystem echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf + echo hostname=\"${VOLUME_LABEL}\" >> ${.TARGET}/etc/rc.conf echo debug.witness.trace=0 >> ${.TARGET}/etc/sysctl.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf cp ${.CURDIR}/rc.local ${.TARGET}/etc -- 2.7.0 From 5acbd64eecca515659d3fc3692e02f7a667aefb0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 19:38:39 +0200 Subject: [PATCH 128/213] Stop bothering packaging docs which are no longer built. Squash or relocate --- Makefile.inc1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index f864ab2..b0e4f7b 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -914,7 +914,7 @@ ITOOLS+=makewhatis # # Non-base distributions produced by the base system -EXTRA_DISTRIBUTIONS= doc +EXTRA_DISTRIBUTIONS= .if defined(LIB32TMP) && ${MK_LIB32} != "no" EXTRA_DISTRIBUTIONS+= lib32 .endif -- 2.7.0 From c0ce2a6f52e8ef7ce0a89110695fe5e5524ef664 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 10 Jun 2015 12:29:37 +0200 Subject: [PATCH 129/213] release/Makefile: Add ${DIST_TARBALL_DIR} ... which contains the distribution tarball directory on the install media. --- release/Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/release/Makefile b/release/Makefile index 15d98dc..8087d06 100644 --- a/release/Makefile +++ b/release/Makefile @@ -60,6 +60,9 @@ DISTDIR= dist NO_ROOT=1 .export NO_ROOT +# Path to the distribution tarballs on the created installation media. +DIST_TARBALL_DIR=usr/freebsd-dist + # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -199,9 +202,9 @@ disc1: packagesystem MK_PROFILE=no MK_SENDMAIL=no MK_TESTS=no MK_LIB32=no \ MK_DEBUG_FILES=no # Copy distfiles - mkdir -p ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} for dist in MANIFEST $$(ls *.txz | grep -vE -- '(base|lib32)-dbg'); \ - do cp $${dist} ${.TARGET}/usr/freebsd-dist; \ + do cp $${dist} ${.TARGET}/${DIST_TARBALL_DIR}; \ done # Copy documentation, if generated .if !defined(NODOC) @@ -229,8 +232,8 @@ bootonly: packagesystem MK_INSTALLIB=no MK_RESCUE=no MK_DICT=no \ MK_KERNEL_SYMBOLS=no MK_TESTS=no MK_DEBUG_FILES=no # Copy manifest only (no distfiles) to get checksums - mkdir -p ${.TARGET}/usr/freebsd-dist - cp MANIFEST ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} + cp MANIFEST ${.TARGET}/${DIST_TARBALL_DIR} # Copy documentation, if generated .if !defined(NODOC) cp reldoc/* ${.TARGET} @@ -250,9 +253,9 @@ dvd: packagesystem DESTDIR=${.OBJDIR}/${.TARGET} MK_RESCUE=no MK_KERNEL_SYMBOLS=no \ MK_TESTS=no MK_DEBUG_FILES=no # Copy distfiles - mkdir -p ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} for dist in MANIFEST $$(ls *.txz | grep -v -- '(base|lib32)-dbg'); \ - do cp $${dist} ${.TARGET}/usr/freebsd-dist; \ + do cp $${dist} ${.TARGET}/${DIST_TARBALL_DIR}; \ done # Copy documentation, if generated .if !defined(NODOC) -- 2.7.0 From becdb40fa2b0570223b9b42c7e5c50c363833ada Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 10 Jun 2015 12:36:37 +0200 Subject: [PATCH 130/213] release/Makefile: ElectroBSDify the shiny new ${DIST_TARBALL_DIR} --- release/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index 8087d06..0675f0e 100644 --- a/release/Makefile +++ b/release/Makefile @@ -61,7 +61,7 @@ NO_ROOT=1 .export NO_ROOT # Path to the distribution tarballs on the created installation media. -DIST_TARBALL_DIR=usr/freebsd-dist +DIST_TARBALL_DIR=usr/electrobsd-dist # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) -- 2.7.0 From 9c2d3dfb5019cd87765565d0f5762c81412c36cc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 15 Jun 2015 12:57:51 +0200 Subject: [PATCH 131/213] Add rc.d script to automatically enable soft-protection on boot --- etc/rc.d/Makefile | 1 + etc/rc.d/soft-protection | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 etc/rc.d/soft-protection diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 1e6e28f..72828d7 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -108,6 +108,7 @@ FILES= DAEMON \ sdpd \ securelevel \ serial \ + soft-protection \ sppp \ ${_sshd} \ statd \ diff --git a/etc/rc.d/soft-protection b/etc/rc.d/soft-protection new file mode 100755 index 0000000..58bc02a --- /dev/null +++ b/etc/rc.d/soft-protection @@ -0,0 +1,46 @@ +#!/bin/sh +# +########################################################################### +# +# soft-protection - Enables cloudiatr soft protection on boot +# +########################################################################### +# +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# PROVIDE: soft_protection +# REQUIRE: FILESYSTEMS + +. /etc/rc.subr + +name="soft_protection" +rcvar="soft_protection_enable" + +soft_protection_enable="${soft_protection_enable-NO}" + +start_cmd="enable_soft_protection" +stop_cmd=":" + +enable_soft_protection() { + # We don't use the soft-protect subcommand + # because it may require user feedback. + cloudiatr cmd cloudiatr_soft_protect +} + +load_rc_config "${name}" +run_rc_command "${1}" -- 2.7.0 From 407d286dd7ca4442ce74e1cf89371ca150500337 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 23:44:16 +0200 Subject: [PATCH 132/213] freebsd-version: Try to work with ElectroBSD and FreeBSD kernels --- bin/freebsd-version/freebsd-version.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/freebsd-version/freebsd-version.sh.in b/bin/freebsd-version/freebsd-version.sh.in index a830d26..630164f 100644 --- a/bin/freebsd-version/freebsd-version.sh.in +++ b/bin/freebsd-version/freebsd-version.sh.in @@ -36,7 +36,7 @@ USERLAND_VERSION="@@REVISION@@-@@BRANCH@@" : ${LOADER_CONF_FILES:=$LOADER_DIR/defaults/loader.conf $LOADER_DIR/loader.conf $LOADER_DIR/loader.conf.local} LOADER_RE1='^\([A-Z_a-z][0-9A-Z_a-z]*=[-./0-9A-Z_a-z]\{1,\}\).*$' LOADER_RE2='^\([A-Z_a-z][0-9A-Z_a-z]*="[-./0-9A-Z_a-z]\{1,\}"\).*$' -KERNEL_RE='^@(#)@@TYPE@@ \([-.0-9A-Za-z]\{1,\}\) .*$' +KERNEL_RE='^@\(#\)(Free|Electro)BSD \([-.0-9A-Za-z]\{1,\}\) .*$' progname=$(basename $0) @@ -67,7 +67,7 @@ kernel_version() { if [ ! -f "$kernfile" -o ! -r "$kernfile" ] ; then error "unable to locate kernel" fi - strings "$kernfile" | sed -n "s/$KERNEL_RE/\\1/p" + strings "$kernfile" | sed -E -n "s/$KERNEL_RE/\\1/p" } # -- 2.7.0 From c43a84a08f717375a4080c5bde9062b13a5425b2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 15 Jun 2015 16:06:25 +0200 Subject: [PATCH 133/213] teken color wip --- sys/dev/syscons/scterm-teken.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/dev/syscons/scterm-teken.c b/sys/dev/syscons/scterm-teken.c index 43f16fa..f3a91fa 100644 --- a/sys/dev/syscons/scterm-teken.c +++ b/sys/dev/syscons/scterm-teken.c @@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -51,6 +52,10 @@ __FBSDID("$FreeBSD$"); #include +static int teken_color=SC_KERNEL_CONS_ATTR; +SYSCTL_INT(_hw, OID_AUTO, teken_color, CTLFLAG_RW, &teken_color, 0, "yolo"); + + static void scteken_revattr(unsigned char, teken_attr_t *); static unsigned int scteken_attr(const teken_attr_t *); @@ -176,7 +181,7 @@ scteken_puts(scr_stat *scp, u_char *buf, int len, int kernel) if (kernel) { /* Use special colors for kernel messages. */ backup = *teken_get_curattr(&ts->ts_teken); - scteken_revattr(SC_KERNEL_CONS_ATTR, &kattr); + scteken_revattr(teken_color, &kattr); teken_set_curattr(&ts->ts_teken, &kattr); teken_input(&ts->ts_teken, buf, len); teken_set_curattr(&ts->ts_teken, &backup); -- 2.7.0 From 926f9c50dde6e535027a8276ff133b72e1e85f1f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 16 Jun 2015 15:21:46 +0200 Subject: [PATCH 134/213] Import geli-key-monitor --- share/dtrace/Makefile | 1 + share/dtrace/README | 2 +- share/dtrace/geli-key-monitor | 191 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 1 deletion(-) create mode 100755 share/dtrace/geli-key-monitor diff --git a/share/dtrace/Makefile b/share/dtrace/Makefile index 83bc48e..4ca76ce 100644 --- a/share/dtrace/Makefile +++ b/share/dtrace/Makefile @@ -17,6 +17,7 @@ SCRIPTS= blocking \ disklatencycmd \ hotopen \ nfsattrstats \ + geli-key-monitor \ nfsclienttime \ siftr \ tcpconn \ diff --git a/share/dtrace/README b/share/dtrace/README index 0540bad..4ef1df6 100644 --- a/share/dtrace/README +++ b/share/dtrace/README @@ -2,7 +2,7 @@ $FreeBSD$ This directory contains scripts for use with the DTrace system. The toolkit/ directory installs the latest vendor import of Brendan -Gregg's DTraceToolkit while the other files and directories +Gregg's DTraceToolkit while the other files and directories mostly contain code generated by the FreeBSD Project for use with DTrace on FreeBSD. diff --git a/share/dtrace/geli-key-monitor b/share/dtrace/geli-key-monitor new file mode 100755 index 0000000..2cf4ba5 --- /dev/null +++ b/share/dtrace/geli-key-monitor @@ -0,0 +1,191 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * geli-key-monitor + * + * Traces GELI to print the beginning of various keys and warns + * about already known ones. For motivation and example output see: + * https://www.fabiankeil.de/gehacktes/geli-key-monitor/ + * + * Copyright (c) 2012 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + algo[ 2] = "3DES-CBC"; + algo[ 3] = "Blowfish-CBC"; + algo[11] = "AES-CBC"; + algo[21] = "Camellia-CBC"; + algo[22] = "AES-XTS"; + + /* + * These are the first bytes of known weak keys generated with + * an unitialized sc->sc_ekey on a little-endian system. + */ + known_keys[0x06d800ef] = 1; + known_keys[0x252c6a6d] = 1; + known_keys[0xa72b9c7c] = 1; + known_keys[0xfde44683] = 1; + + G_ELI_FLAG_ENC_IVKEY = 0x00400000; + + g_eli_key_stats_available = 0; + g_eli_hold_stats_available = 0; + + /* + * Geli v5 and higher use a different key for each GB, + * so large disks have several thousand keys. + * Monitoring only the first few should do, keeps the + * output and reduces the likelihood of prevents dynvar + * drops. + */ + max_disk_keys_to_monitor = 4; + + printf("%Y: Monitoring geli keys (up to %d encryption keys per disk). %s\n", + walltimestamp, max_disk_keys_to_monitor, "Press CTRL-C to exit."); +} + +fbt::g_eli_mkey_propagate:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->mkey = (struct g_eli_key *)arg1; + + self->geom_name = stringof(self->sc->sc_geom->name); + self->monitored_keys = 0; + self->key_limit_reached = 0; + + this->algo = self->sc->sc_ealgo; + this->algo_name = (algo[this->algo] != NULL) ? algo[this->algo] : "Unregistered"; + this->geli_version = self->sc->sc_version; + + printf("%Y: %s: %s:%-6s: version: %d, algo: %s, flags: 0x%x (FLAG_ENC_IVKEY: %d) ", + walltimestamp, + self->geom_name, + probefunc, probename, + this->geli_version, this->algo_name, + self->sc->sc_flags, + (self->sc->sc_flags & G_ELI_FLAG_ENC_IVKEY) != 0 + ); + printf("mkey: %08.8x, sc_mkey: %08.8x, sc_ekey: %08.8x\n", + *(uint32_t *)(self->mkey), + *(uint32_t *)(self->sc->sc_mkey), + *(uint32_t *)(self->sc->sc_ekey)); +} + +fbt::g_eli_mkey_propagate:return +/self->sc != NULL/ +{ + printf("%Y: %s: %s:%-6s: mkey: %08.8x, sc_mkey: %08.8x, sc_ekey: %08.8x, sc_ekeys_allocated: %d\n", + walltimestamp, + self->geom_name, + probefunc, probename, + *(uint32_t *)(self->mkey), + *(uint32_t *)(self->sc->sc_mkey), + *(uint32_t *)(self->sc->sc_ekey), + self->sc->sc_ekeys_allocated); +} + +fbt::g_eli_key_fill:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->key = (struct g_eli_key *)arg1; + self->geom_name = stringof(self->sc->sc_geom->name); +} + +fbt::g_eli_key_fill: +/(self->sc != NULL) && !self->key_limit_reached/ +{ + this->key = *(uint32_t *)(self->key->gek_key); + printf("%Y: %s: %s:%-6s: key->gek_key: %8.8x (%d).%s\n", + walltimestamp, + self->geom_name, + probefunc, probename, + this->key, + self->monitored_keys, + known_keys[this->key] ? " Key looks familiar!" : ""); +} + +fbt::g_eli_key_fill:return +/(self->sc != NULL) && !self->key_limit_reached/ +{ + this->key = *(uint32_t *)(self->key->gek_key); + @g_eli_keys[self->monitored_keys, this->key, self->geom_name] = count(); + /* + * Register the generated key as known. + * Unless the provider is reattached we do not want to see it again. + */ + known_keys[this->key] = 1; + g_eli_key_stats_available = 1; + self->monitored_keys++; +} + +fbt::g_eli_key_fill:return +/(self->sc != NULL) && (self->sc->sc_ekeys_allocated == max_disk_keys_to_monitor - 1)/ +{ + printf("%Y: %s: %s:%-6s: Encryption key limit per disk reached.\n", + walltimestamp, self->geom_name, probefunc, probename); + self->key_limit_reached = 1; +} + +fbt::g_eli_key_hold:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->offset = (off_t)arg1; + self->blocksize = (size_t)arg2; + self->geom_name = stringof(self->sc->sc_geom->name); + + this->first_mkey_bytes = *(uint32_t *)(self->sc->sc_mkey); + this->first_ekey_bytes = *(uint32_t *)(self->sc->sc_ekey); + /* + * We only check the first bytes here, so false-positives are + * theoretically posible although unlikely. + */ + this->empty_ekey = (0 == this->first_ekey_bytes); + + this->geli_version = self->sc->sc_version; + this->algo = self->sc->sc_ealgo; + this->algo_name = (algo[this->algo] != NULL) ? algo[this->algo] : "Unregistered"; + + @g_eli_key_hold[self->geom_name, + this->geli_version, + this->algo_name, + this->algo, + this->first_mkey_bytes, + this->first_ekey_bytes] = count(); + + g_eli_hold_stats_available = 1; +} + +tick-60sec, +dtrace:::END +/g_eli_hold_stats_available/ +{ + printf("\n---------------------------------------------------------\n"); + printf("%Y: g_eli_key_hold() calls so far:\n", walltimestamp); + printf("%-20s %12s %18s %11s %11s %10s\n", + "Provider", "Geli version", "Algorithm", "mkey start", "ekey start", "calls"); + printa("%-20s %12d %13s (%2d) %08x %08x %@10d\n", @g_eli_key_hold); +} + +tick-60sec, +dtrace:::END +/g_eli_key_stats_available/ +{ + printf("\n%Y: g_eli_key count (only works for geli version 5 or higher):\n", walltimestamp); + printa("%@u #%04d %8.8x %-25s\n", @g_eli_keys); +} -- 2.7.0 From e6100395590d36a0ab0696b6cab1f833306c702b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 18 Jun 2015 12:52:20 +0200 Subject: [PATCH 135/213] jemalloc: Enable MALLOC_PRODUCTION --- contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h b/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h index 1ab2ce5..f91710e 100644 --- a/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h +++ b/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h @@ -4,6 +4,8 @@ #undef JEMALLOC_OVERRIDE_VALLOC +#define MALLOC_PRODUCTION + #ifndef MALLOC_PRODUCTION #define JEMALLOC_DEBUG #endif -- 2.7.0 From 4478499d53bd5801b26cce05278d4be0adc09613 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 16:46:35 +0200 Subject: [PATCH 136/213] release/Makefile: Note that the release process may silently fail --- release/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release/Makefile b/release/Makefile index 0675f0e..ade5dcb 100644 --- a/release/Makefile +++ b/release/Makefile @@ -2,6 +2,14 @@ # # Makefile for building releases and release media. # +# Note that the release process is rather fragile and lots +# of variables depend on each other in mysterious and +# undocumented ways. +# +# If you don't get the magic right you should consider yourself +# lucky if the build fails, the usual outcome is a release that +# does not work. +# # User-driven targets: # cdrom: Builds release CD-ROM media (disc1.iso) # dvdrom: Builds release DVD-ROM media (dvd1.iso) -- 2.7.0 From 8f70c564f043d8fd0e18f0840565882da403c176 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 16:50:54 +0200 Subject: [PATCH 137/213] release/Makefile: Remove more stuff that does not apply to ElecroBSD from the disc1 target --- release/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/release/Makefile b/release/Makefile index ade5dcb..724614c 100644 --- a/release/Makefile +++ b/release/Makefile @@ -219,12 +219,10 @@ disc1: packagesystem cp reldoc/* ${.TARGET} .endif # Set up installation environment - ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf echo hostname=\"${VOLUME_LABEL}\" >> ${.TARGET}/etc/rc.conf - echo debug.witness.trace=0 >> ${.TARGET}/etc/sysctl.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.7.0 From ec5e2d2b31de3765ca70efbe10c378261bf3a6b0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 17:08:38 +0200 Subject: [PATCH 138/213] Makefile.inc1: Workaround passwd and friends missing in the base.txz --- Makefile.inc1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index b0e4f7b..2e7cfd2 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1018,6 +1018,9 @@ distributeworld installworld: _installcheck_world -e 's@(type=)@uid=0 gid=0 time=${EPOCH_DATE}.0 \1@' \ -e 's@//@/@' -i '.bak' ${METALOG} .endif + @# Workaround for parts of etc mysteriously not being added below base. + @# As usual the location in dist is fine. + sed -e 's@^\./etc@./base/etc@' -i '.etc.bak' ${METALOG} .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that -- 2.7.0 From 3fca4503778dac74be1ff6d0c6711cef26007658 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 27 Jun 2015 19:16:04 +0200 Subject: [PATCH 139/213] ata_da: Ditch support for legacy device names ... and remove the annoying "Previously was known as adX" messages. (The code no longer matches the commit message precisely as upstream had a similar idea a while later and just missed a block which prevented the commit from comletely disappearing while rebasing) --- sys/cam/ata/ata_da.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c index 6ea25c2..b4bd773 100644 --- a/sys/cam/ata/ata_da.c +++ b/sys/cam/ata/ata_da.c @@ -535,10 +535,6 @@ static void adashutdown(void *arg, int howto); static void adasuspend(void *arg); static void adaresume(void *arg); -#ifndef ADA_DEFAULT_LEGACY_ALIASES -#define ADA_DEFAULT_LEGACY_ALIASES 1 -#endif - #ifndef ADA_DEFAULT_TIMEOUT #define ADA_DEFAULT_TIMEOUT 30 /* Timeout in seconds */ #endif -- 2.7.0 From 3493e5d760318b40fe7a681f62ec150814fdaa30 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 13:45:09 +0200 Subject: [PATCH 140/213] Use make-memstick.sh's amd64 version on i386 as well --- release/i386/make-memstick.sh | 44 +------------------------------------------ 1 file changed, 1 insertion(+), 43 deletions(-) mode change 100755 => 120000 release/i386/make-memstick.sh diff --git a/release/i386/make-memstick.sh b/release/i386/make-memstick.sh deleted file mode 100755 index f824382..0000000 --- a/release/i386/make-memstick.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# This script generates a "memstick image" (image that can be copied to a -# USB memory stick) from a directory tree. Note that the script does not -# clean up after itself very well for error conditions on purpose so the -# problem can be diagnosed (full filesystem most likely but ...). -# -# Usage: make-memstick.sh -# -# $FreeBSD$ -# - -PATH=/bin:/usr/bin:/sbin:/usr/sbin -export PATH - -if [ $# -ne 2 ]; then - echo "make-memstick.sh /path/to/directory /path/to/image/file" - exit 1 -fi - -if [ ! -d ${1} ]; then - echo "${1} must be a directory" - exit 1 -fi - -if [ -e ${2} ]; then - echo "won't overwrite ${2}" - exit 1 -fi - -echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab -echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -makefs -B little -o label=FreeBSD_Install ${2}.part ${1} -if [ $? -ne 0 ]; then - echo "makefs failed" - exit 1 -fi -rm ${1}/etc/fstab -rm ${1}/etc/rc.conf.local - -mkimg -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} -rm ${2}.part - diff --git a/release/i386/make-memstick.sh b/release/i386/make-memstick.sh new file mode 120000 index 0000000..978e04a --- /dev/null +++ b/release/i386/make-memstick.sh @@ -0,0 +1 @@ +../amd64/make-memstick.sh \ No newline at end of file -- 2.7.0 From 3a5b26c75547ddd380f78c846df2567705633e5f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 09:50:52 +0200 Subject: [PATCH 141/213] copyright.h: Use more insightful trademark information --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index 12d421e..a6121f7 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -38,7 +38,7 @@ /* Foundation */ #define TRADEMARK_Foundation \ - "FreeBSD is a registered trademark of The FreeBSD Foundation.\n" + "ElectroBSD ain't no registered trademark of The ElectroBSD Foundation (which does not exist).\n" /* Berkeley */ #define COPYRIGHT_UCB \ -- 2.7.0 From 3dc787e49b09c32778a0283555c3ea4f7a8155ff Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 09:54:11 +0200 Subject: [PATCH 142/213] g_new_provider_event(): Return early if the provider is already withering Previously debug kernels would panic, other side effects haven't been diagnosed yet. Patch by Scott M. Ferris. FreeBSD bug: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=200740 --- sys/geom/geom_subr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/geom/geom_subr.c b/sys/geom/geom_subr.c index bf14a86..ec6ed4d 100644 --- a/sys/geom/geom_subr.c +++ b/sys/geom/geom_subr.c @@ -533,8 +533,8 @@ g_new_provider_event(void *arg, int flag) return; pp = arg; G_VALID_PROVIDER(pp); - KASSERT(!(pp->flags & G_PF_WITHER), - ("g_new_provider_event but withered")); + if ((pp->flags & G_PF_WITHER) != 0) + return; LIST_FOREACH_SAFE(cp, &pp->consumers, consumers, next_cp) { if ((cp->flags & G_CF_ORPHAN) == 0 && cp->geom->attrchanged != NULL) -- 2.7.0 From 536b12db4bfcbbd7e16f33623b8194c782f990fa Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 8 Jul 2015 17:43:31 +0200 Subject: [PATCH 143/213] sys/kern: Remove another FreeBSD reference from the boot messages --- sys/kern/subr_smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 82349f8..ae08a8a 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -154,7 +154,7 @@ mp_start(void *dummy) } cpu_mp_start(); - printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", + printf("SMP: Multiprocessor System Detected: %d CPUs\n", mp_ncpus); cpu_mp_announce(); } -- 2.7.0 From 5a9f7a28f2b19181b38d601dc28c76df6da84bd4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 12 Jul 2015 13:11:28 +0200 Subject: [PATCH 144/213] tools/test/devrandom: Import arctest d9a5fc80, a wrapper around dieharder --- tools/test/devrandom/arc4test | 278 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100755 tools/test/devrandom/arc4test diff --git a/tools/test/devrandom/arc4test b/tools/test/devrandom/arc4test new file mode 100755 index 0000000..c824997 --- /dev/null +++ b/tools/test/devrandom/arc4test @@ -0,0 +1,278 @@ +#!/bin/sh + +############################################################################ +# arc4test +# +# Collects "entropy" and lets dieharder analyze it later on. The collected +# entropy is split into smaller files so the data collected in multiple +# runs can be easily interleaved and tested together. +# +# By default, entropy files are generated with arc4cat, a wrapper around +# arc4random_buf(3) which is suspected of "not returning very random data" +# between FreeBSD r273872 and r278907. +# +# So far it looks like the data may be "random enough" to pass the tests. +# +# Usage: +# arc4test build : Build arc4cat in $ARC4CAT_DIR +# arc4test collect : Collect potential entropy with arc4cat +# arc4test collect -d : Collect potential entropy with Dilbert PNRG +# arc4test analyze : Interleave collected entropy files and +# pipe them into dieharder. +# arc4test analyze -f : Try to spead up things by caching the interleaved +# entropy in a single file. Reuses the file if it +# already exists. +# arc4test remix : (Re)build an entropy cache file based on the +# previously collected entropy files. Roughfly +# doubles the required disk space but may significantly +# improve performance. +# arc4test cat : Dump interleaved entropy files to stdout +# +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +############################################################################ + +ARC4CAT_DIR=. +ARC4CAT="${ARC4CAT_DIR}/arc4cat" +# May not contain spaces etc. +ENTROPY_DIR="./entropy" +ENTROPY_SUBDIR_PREFIX="collection-" +# This is currently the block size for dd, setting it "too high" +# is not expected to work, however that's not good for input mixing +# later on anyway. +# +# Currently the entropy is split while it's being collected which is +# convenient from a programming point of view, but if the file size +# is small the interleave performance "may" (vulgo: will) suck. +ENTROPY_FILE_SIZE=4k +# Number of entropy files collected per run. If you increase ENTROPY_FILE_SIZE +# (or intend to do lots of collection runs) you may want to decrease this. +MAX_ENTROPY_FILES=100000 +ENTROPY_CACHE_FILE="${ENTROPY_DIR}/cached-entropy" + +prepare() { +} + +# The default entropy source +get_arc4cat_entropy() { + "${ARC4CAT}" +} + +# This is the reverse engineered PRNG from Dilbert strip 2001-10-25: +# http://dilbert.com/strip/2001-10-25 +# +# It is used instead of get_arc4cat_entropy() if the +# collect flag -d is set. +# +# According to the literature (see URL above) you can never be sure +# if the output is random, however the generator seems to fail all +# the dieharder tests and thus doesn't look nearly as good as Yarrow +# and Fortuna. +# +# Until this changes it will not be considered for ElectroBSD. +get_dilbert_entropy() { + while true; do + # The loop has been partially unrolled + # for increased performance! + echo -n "999999" + done +} + +get_shiny_new_entropy() { + local entropy_flag="${1}" + + if [ "${entropy_flag}" = "-d" ]; then + get_dilbert_entropy + else + get_arc4cat_entropy + fi +} + +collect_entropy() { + local entropy_flag \ + i entropy_file entropy_subdir + + entropy_flag="${1}" + i=0 + entropy_subdir="${ENTROPY_DIR}/${ENTROPY_SUBDIR_PREFIX}$(date +%s)" + + mkdir -p "${entropy_subdir}" + + ENTROPY_FILE_POSTFIX="" + + echo "Collecting ${MAX_ENTROPY_FILES} entropy files of size ${ENTROPY_FILE_SIZE} ..." + + # We don't call get_shiny_new_entropy() inside the loop as it + # would result in bits of entropy getting dropped on the floor + # between files. While we don't care about the "waste", we do + # care about not being able to test those bits later on. + get_shiny_new_entropy "${entropy_flag}" | while [ "${i}" -lt "${MAX_ENTROPY_FILES}" ]; do + entropy_file="$(printf "${entropy_subdir}/%.6i" "${i}")" + #echo "Creating ${entropy_file}" + dd bs="${ENTROPY_FILE_SIZE}" count=1 of="${entropy_file}" 2>/dev/null + i=$((i + 1)) + done +} + +create_entropy_cache() { + local \ + entropy_file + + entropy_file="${ENTROPY_DIR}/${LARGE_ENTROPY_FILE_NAME}" + + echo "Building a single entropy file '${ENTROPY_CACHE_FILE}' based on the files collected previously ..." 1>&2 + cat_collected_entropy > "${ENTROPY_CACHE_FILE}" +} + +replay_entropy() { + local fast_flag \ + entropy_file + + fast_flag="${1}" + + if [ "${fast_flag}" = "-f" ]; then + entropy_file="${ENTROPY_DIR}/${LARGE_ENTROPY_FILE_NAME}" + + if [ ! -f "${ENTROPY_CACHE_FILE}" ]; then + create_entropy_cache + fi + cat_entropy_cache + else + cat_collected_entropy + fi +} + +warn_about_entropy_reuse() { + echo "$0: Oh noes, we're out of collected entropy. Going back to the beginning." 1>&2 + echo "This shouldn't be a problem as long as no single test sees repeated data." 1>&2 +} + +cat_entropy_cache() { + while true; do + cat "${ENTROPY_CACHE_FILE}" + warn_about_entropy_reuse + done +} + +cat_collected_entropy() { + local \ + i f entropy_collections entropy_subdir entropy_file + + # XXX: Too fucking slow + #entropy_collections="$(find "${ENTROPY_DIR}/" -name "${ENTROPY_SUBDIR_PREFIX}*" -depth 1 -type 1)" + + # Not best practice but at least the performance doesn't suck + # and it works as expected. + + entropy_collections="${ENTROPY_DIR}/${ENTROPY_SUBDIR_PREFIX}"* + i=0 + while true; do + f="$(printf "%.6i" "${i}")" + for entropy_subdir in $entropy_collections; do + entropy_file="${entropy_subdir}/${f}" + #echo "Catting ${entropy_file}" + cat "${entropy_file}" || return 1 + done + i=$((i + 1)) + if [ "${i}" -eq "${MAX_ENTROPY_FILES}" ]; then + warn_about_entropy_reuse + i=0 + fi + done +} + +get_dieharder_tests() { + dieharder -l | awk '/-d/ {print $2}' +} + +analyze_collected_entropy() { + local fast_flag \ + test_number + + fast_flag="${1}" + + # We call replay_entropy() inside the loop to make sure + # the beginning of the collected entropy is checked by all tests + # (instead of having each test start at different offsets). + for test_number in $(get_dieharder_tests); do + replay_entropy ${fast_flag} | dieharder -g 200 -d "${test_number}" + done +} + +get_arc4cat_code() { + cat< +#include +#include + +int main(void) { + char buf[4096]; + + while (1) { + arc4random_buf(buf, sizeof(buf)); + write(1, buf, sizeof(buf)); + } +} +EOF +} + +build_arc4cat() { + mkdir -p "${ARC4CAT_DIR}" + cd "${ARC4CAT_DIR}" + get_arc4cat_code > arc4cat.c + make arc4cat + rm arc4cat.c +} + +usage() { + echo "Looks like you are doing it wrong. Try one of these:" + echo + echo "$0 analyze" + echo "$0 build" + echo "$0 cat" + echo "$0 collect" + return 1 +} + +main() { + local mode="${1}" + + shift + set -e + prepare + + case "${mode}" in + analyze) + analyze_collected_entropy "${@}" + ;; + build) + build_arc4cat + ;; + cat) + cat_collected_entropy + ;; + collect) + collect_entropy "${@}" + ;; + remix) + create_entropy_cache + ;; + *) + usage + ;; + esac +} + +main "${@}" -- 2.7.0 From 05c48696f3cd9c62fceab51044ac10066799c7e5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 23 Jul 2015 19:16:44 +0200 Subject: [PATCH 145/213] dtrace_consume(): Warn if cpu cores aren't completely in sync ... instead of aborting. While the results may be not completely accurate, in some cases it may not matter. Previously the assertion would sometimes be triggered on a systems with poor timecounters (TSC-low(-100) ACPI-fast(900) i8254(0) dummy(-1000000)). --- cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c index 04c022b..39530cf 100644 --- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c +++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -3325,7 +3326,13 @@ dtrace_consume(dtrace_hdl_t *dtp, FILE *fp, if (buf != NULL) { if (first_timestamp == 0) first_timestamp = buf->dtbd_timestamp; - assert(buf->dtbd_timestamp >= first_timestamp); + if (buf->dtbd_timestamp < first_timestamp) { + warnx("cpu clocks out of sync " + "(%ju < %ju; offset: %ju). " + "Results may be incorrect!", + buf->dtbd_timestamp, first_timestamp, + first_timestamp - buf->dtbd_timestamp); + } dt_pq_insert(dtp->dt_bufq, buf); drops[i] = buf->dtbd_drops; -- 2.7.0 From aec3bbfb8e6d94fd6f6993ddd7b31ded7bcabc1b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 19:06:37 +0200 Subject: [PATCH 146/213] release: Add ElectroBSD install instructions ... that aren't particularly verbose, though. --- release/rc.local | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/release/rc.local b/release/rc.local index cb44227..0d2fad2 100755 --- a/release/rc.local +++ b/release/rc.local @@ -37,7 +37,7 @@ if [ $? -eq 0 ]; then else # Serial or other console echo - echo "Welcome to FreeBSD!" + echo "Welcome to ElectroBSD!" echo echo "Please choose the appropriate terminal type for your system." echo "Common console types are:" @@ -62,7 +62,19 @@ if [ -f /etc/installerconfig ]; then exit fi -dialog --backtitle "FreeBSD Installer" --title "Welcome" --extra-button --extra-label "Shell" --ok-label "Install" --cancel-label "Live CD" --yesno "Welcome to FreeBSD! Would you like to begin an installation or use the live CD?" 0 0 +local timeout=15 +if dialog --backtitle "ElectroBSD $(uname -m) on a stick in da house" \ + --title "Careful now, your data may be at risk." \ + --timeout ${timeout} \ + --yesno "To install ElectroBSD, execute cloudiatr after creating a cloudiatr.conf. Understood? You have ${timeout} seconds to respond." \ + 0 0; then + echo "Great. Good luck." +else + echo "That's very unfortunate. If you don't have backups you probably should not continue." +fi + +exit 0; + case $? in $DIALOG_OK) # Install -- 2.7.0 From 3563d5dee4b411918cbdbb96d780f20eed3d40d7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 28 Jul 2015 15:07:52 +0200 Subject: [PATCH 147/213] etc/rc.d/dumpon: Get dumpdev=AUTO working for GEOM_ELI consumers --- etc/rc.d/dumpon | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/rc.d/dumpon b/etc/rc.d/dumpon index ce5fc1c..f1f3bcb 100755 --- a/etc/rc.d/dumpon +++ b/etc/rc.d/dumpon @@ -41,6 +41,7 @@ dumpon_start() fi while read dev mp type more ; do [ "${type}" = "swap" ] || continue + dev="${dev%%.eli}" [ -c "${dev}" ] || continue dumpon_try "${dev}" 2>/dev/null && return 0 done Date: Tue, 16 Jun 2015 15:07:06 +0200 Subject: [PATCH 148/213] deadlkres(): (Try to) optionally unlock deadlocked processes to work around USB deadlocks This is work in progress and hasn't properly tested yet. --- sys/kern/kern_clock.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index fdf2271..81228a5 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -178,6 +178,7 @@ static const char *blessed[] = { }; static int slptime_threshold = 1800; static int blktime_threshold = 900; +static int unlock_deadlocked_processes = 0; static int sleepfreq = 3; static void @@ -236,9 +237,15 @@ deadlkres(void) * turnstile. */ PROC_UNLOCK(p); - sx_sunlock(&allproc_lock); + if (unlock_deadlocked_processes == 0) { + sx_sunlock(&allproc_lock); panic("%s: possible deadlock detected for %p, blocked for %d ticks\n", - __func__, td, tticks); + __func__, td, tticks); + } else { + printf("%s: possible deadlock detected for %p, blocked for %d ticks. " + "Unlocking process to see what happens. Good luck.\n", __func__, td, tticks); + TD_CLR_LOCK(td); + } } } else if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) { @@ -282,9 +289,15 @@ deadlkres(void) continue; } PROC_UNLOCK(p); - sx_sunlock(&allproc_lock); + if (unlock_deadlocked_processes == 0) { + sx_sunlock(&allproc_lock); panic("%s: possible deadlock detected for %p, blocked for %d ticks\n", - __func__, td, tticks); + __func__, td, tticks); + } else { + printf("%s: possible deadlock detected for %p, blocked for %d ticks. " + "Unlocking process to see what happens. Good luck.\n", __func__, td, tticks); + TD_CLR_LOCK(td); + } } } else thread_unlock(td); @@ -316,6 +329,11 @@ SYSCTL_INT(_debug_deadlkres, OID_AUTO, blktime_threshold, CTLFLAG_RW, "Number of seconds within is valid to block on a turnstile"); SYSCTL_INT(_debug_deadlkres, OID_AUTO, sleepfreq, CTLFLAG_RW, &sleepfreq, 0, "Number of seconds between any deadlock resolver thread run"); +SYSCTL_INT(_debug_deadlkres, OID_AUTO, + unlock_deadlocked_processes_and_see_what_happens, + CTLFLAG_RW, &unlock_deadlocked_processes, 0, + "'Resolve' deadlocks by merely unlocking the locked process. " + "May cause permanent data corruption."); #endif /* DEADLKRES */ void -- 2.7.0 From 1cb4c1bd4777fefb2c97c82d93826306b5ed17a1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 1 Aug 2015 11:42:55 +0200 Subject: [PATCH 149/213] sys/dev/vt: Default to showing splash CPU logo(s) on boot --- sys/dev/vt/vt_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index 40be782..62a99f9 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -137,7 +137,7 @@ static VT_SYSCTL_INT(kbd_panic, 0, "Enable request to panic. " /* Used internally, not a tunable. */ int vt_draw_logo_cpus; -VT_SYSCTL_INT(splash_cpu, 0, "Show logo CPUs during boot"); +VT_SYSCTL_INT(splash_cpu, 1, "Show logo CPUs during boot"); VT_SYSCTL_INT(splash_ncpu, 0, "Override number of logos displayed " "(0 = do not override)"); VT_SYSCTL_INT(splash_cpu_style, 2, "Draw logo style " -- 2.7.0 From 4aa0f8ba0c0ad8e290940a4dc70e03d41cc6ec2f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 1 Aug 2015 16:59:55 +0200 Subject: [PATCH 150/213] sys/dev/vt: Default to showing Beastie instead of the orb/sextoy --- sys/dev/vt/vt_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index 62a99f9..dfc7a32 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -140,7 +140,7 @@ int vt_draw_logo_cpus; VT_SYSCTL_INT(splash_cpu, 1, "Show logo CPUs during boot"); VT_SYSCTL_INT(splash_ncpu, 0, "Override number of logos displayed " "(0 = do not override)"); -VT_SYSCTL_INT(splash_cpu_style, 2, "Draw logo style " +VT_SYSCTL_INT(splash_cpu_style, 1, "Draw logo style " "(0 = Alternate beastie, 1 = Beastie, 2 = Orb)"); VT_SYSCTL_INT(splash_cpu_duration, 10, "Hide logos after (seconds)"); -- 2.7.0 From 8b025f2b1629e3084123d8ace4f3c6d2e1889d8e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 2 Aug 2015 17:55:17 +0200 Subject: [PATCH 151/213] sys/modules: Detach reiserfs from the build. All hail OpenZFS --- sys/modules/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 76a12d3..32aaab4 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -309,7 +309,6 @@ SUBDIR= \ ${_rdma} \ ${_rdrand_rng} \ re \ - reiserfs \ rl \ rtwn \ ${_rtwnfw} \ -- 2.7.0 From 32655ce6b113f144d1cdad05b65e5687cae9b0e4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 2 Aug 2015 17:56:01 +0200 Subject: [PATCH 152/213] sys/modules: Detach a couple of ATM-related modules --- sys/modules/Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 32aaab4..a834873 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -110,7 +110,6 @@ SUBDIR= \ ${_ex} \ ${_exca} \ ext2fs \ - ${_fatm} \ fdc \ fdescfs \ ${_fe} \ @@ -123,7 +122,6 @@ SUBDIR= \ geom \ ${_glxiic} \ ${_glxsb} \ - hatm \ hifn \ hme \ ${_hpt27xx} \ @@ -277,7 +275,6 @@ SUBDIR= \ ow \ ${_padlock} \ ${_padlock_rng} \ - patm \ ${_pccard} \ ${_pcfclock} \ pcn \ @@ -478,7 +475,6 @@ _pfsync= pfsync .if ${MK_SOURCELESS_UCODE} != "no" _bce= bce -_fatm= fatm _fxp= fxp _ispfw= ispfw _mwlfw= mwlfw -- 2.7.0 From d2bd46333b67b205ea474783f157c8f897b55226 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 2 Aug 2015 19:41:56 +0200 Subject: [PATCH 153/213] crypto/openssl: Update OPENSSL_VERSION_TEXT and break 'FIPS mode' --- crypto/openssl/crypto/opensslv.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/openssl/crypto/opensslv.h b/crypto/openssl/crypto/opensslv.h index b315d5e..f8c05c1 100644 --- a/crypto/openssl/crypto/opensslv.h +++ b/crypto/openssl/crypto/opensslv.h @@ -32,9 +32,9 @@ extern "C" { */ # define OPENSSL_VERSION_NUMBER 0x1000206fL # ifdef OPENSSL_FIPS -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2f-fips 28 Jan 2016" +# error "FIPS validation is considered silly and thus not supported by ElectroBSD." # else -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2f-freebsd 28 Jan 2016" +# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2f-ElectroBSD 2016-01-28" # endif # define OPENSSL_VERSION_PTEXT " part of " OPENSSL_VERSION_TEXT -- 2.7.0 From 535b7865d8ec74f48c59ba04728be03f8469be2d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 13:13:09 +0200 Subject: [PATCH 154/213] libstand: Show a message instead of the boring twiddle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obviously the message should be sysctl controlled, but for now it's just a hardcoded 'Die Ärzte' reference which may contain traces of electro beer. --- lib/libstand/twiddle.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/libstand/twiddle.c b/lib/libstand/twiddle.c index 96ebbbe..b8c1117 100644 --- a/lib/libstand/twiddle.c +++ b/lib/libstand/twiddle.c @@ -48,6 +48,7 @@ void twiddle(u_int callerdiv) { static u_int callercnt, globalcnt, pos; + static const char message[] = "5,6,7,8 - Bullenstaat!"; callercnt++; if (callerdiv > 1 && (callercnt % callerdiv) != 0) @@ -57,8 +58,9 @@ twiddle(u_int callerdiv) if (globaldiv > 1 && (globalcnt % globaldiv) != 0) return; - putchar("|/-\\"[pos++ & 3]); - putchar('\b'); + putchar((pos < sizeof(message)) ? message[pos] : '\b'); + pos++; + pos = pos % sizeof(message); } void -- 2.7.0 From fd795a996708d3252e1f6da16c44b92af48febdc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 4 Aug 2015 12:35:43 +0200 Subject: [PATCH 155/213] libstand: Change twiddle message to Elektrobier --- lib/libstand/twiddle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libstand/twiddle.c b/lib/libstand/twiddle.c index b8c1117..f234497 100644 --- a/lib/libstand/twiddle.c +++ b/lib/libstand/twiddle.c @@ -48,7 +48,7 @@ void twiddle(u_int callerdiv) { static u_int callercnt, globalcnt, pos; - static const char message[] = "5,6,7,8 - Bullenstaat!"; + static const char message[] = "Elektrobier!\n"; callercnt++; if (callerdiv > 1 && (callercnt % callerdiv) != 0) -- 2.7.0 From 05e5edacaa2662d3bdff115ebb51cb0919aecd63 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 11 Aug 2015 19:16:01 +0200 Subject: [PATCH 156/213] sys/kern/kern_fork: Prevent inlining of a couple of fork related functions to make debugging poudriere crashes more convenient --- sys/kern/kern_fork.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index e7d7276..a6f30e2 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -213,7 +213,7 @@ sysctl_kern_randompid(SYSCTL_HANDLER_ARGS) SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_kern_randompid, "I", "Random PID modulus"); -static int +static __noinline int fork_findpid(int flags) { struct proc *p; @@ -315,7 +315,7 @@ again: return (trypid); } -static int +static __noinline int fork_norfproc(struct thread *td, int flags) { int error; @@ -365,7 +365,7 @@ fail: return (error); } -static void +static __noinline void do_fork(struct thread *td, int flags, struct proc *p2, struct thread *td2, struct vmspace *vm2, int pdflags) { -- 2.7.0 From 8a9bcce023009521ca6725a5dd7152534395899a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 15 Aug 2015 11:29:39 +0200 Subject: [PATCH 157/213] ttys: Reduce default number of active terminals and mark the console as insecure --- etc/etc.amd64/ttys | 16 ++++++++-------- etc/etc.i386/ttys | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/etc/etc.amd64/ttys b/etc/etc.amd64/ttys index 15eb30d..457b3d1 100644 --- a/etc/etc.amd64/ttys +++ b/etc/etc.amd64/ttys @@ -27,17 +27,17 @@ # # If console is marked "insecure", then init will ask for the root password # when going to single-user mode. -console none unknown off secure +console none unknown off insecure # ttyv0 "/usr/libexec/getty Pc" xterm on secure # Virtual terminals -ttyv1 "/usr/libexec/getty Pc" xterm on secure -ttyv2 "/usr/libexec/getty Pc" xterm on secure -ttyv3 "/usr/libexec/getty Pc" xterm on secure -ttyv4 "/usr/libexec/getty Pc" xterm on secure -ttyv5 "/usr/libexec/getty Pc" xterm on secure -ttyv6 "/usr/libexec/getty Pc" xterm on secure -ttyv7 "/usr/libexec/getty Pc" xterm on secure +ttyv1 "/usr/libexec/getty Pc" xterm off secure +ttyv2 "/usr/libexec/getty Pc" xterm off secure +ttyv3 "/usr/libexec/getty Pc" xterm off secure +ttyv4 "/usr/libexec/getty Pc" xterm off secure +ttyv5 "/usr/libexec/getty Pc" xterm off secure +ttyv6 "/usr/libexec/getty Pc" xterm off secure +ttyv7 "/usr/libexec/getty Pc" xterm off secure ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure # Serial terminals # The 'dialup' keyword identifies dialin lines to login, fingerd etc. diff --git a/etc/etc.i386/ttys b/etc/etc.i386/ttys index 15eb30d..457b3d1 100644 --- a/etc/etc.i386/ttys +++ b/etc/etc.i386/ttys @@ -27,17 +27,17 @@ # # If console is marked "insecure", then init will ask for the root password # when going to single-user mode. -console none unknown off secure +console none unknown off insecure # ttyv0 "/usr/libexec/getty Pc" xterm on secure # Virtual terminals -ttyv1 "/usr/libexec/getty Pc" xterm on secure -ttyv2 "/usr/libexec/getty Pc" xterm on secure -ttyv3 "/usr/libexec/getty Pc" xterm on secure -ttyv4 "/usr/libexec/getty Pc" xterm on secure -ttyv5 "/usr/libexec/getty Pc" xterm on secure -ttyv6 "/usr/libexec/getty Pc" xterm on secure -ttyv7 "/usr/libexec/getty Pc" xterm on secure +ttyv1 "/usr/libexec/getty Pc" xterm off secure +ttyv2 "/usr/libexec/getty Pc" xterm off secure +ttyv3 "/usr/libexec/getty Pc" xterm off secure +ttyv4 "/usr/libexec/getty Pc" xterm off secure +ttyv5 "/usr/libexec/getty Pc" xterm off secure +ttyv6 "/usr/libexec/getty Pc" xterm off secure +ttyv7 "/usr/libexec/getty Pc" xterm off secure ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure # Serial terminals # The 'dialup' keyword identifies dialin lines to login, fingerd etc. -- 2.7.0 From 13bb2e94bd54d15121edeedeca913f671f2acb0f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 21:28:15 +0200 Subject: [PATCH 158/213] usr.sbin: Don't build uathload if MK_SOURCELESS_UCODE is defined. It breaks the build if the source tree does not contain the firmware --- usr.sbin/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index 86b5247..b5e1961 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -194,9 +194,14 @@ SUBDIR.${MK_TIMED}+= timed SUBDIR.${MK_TOOLCHAIN}+= config SUBDIR.${MK_TOOLCHAIN}+= crunch SUBDIR.${MK_UNBOUND}+= unbound +# XXX: MK_SOURCELESS_UCODE is supposed to affect kernel +# modules only but there is no matching define for +# userland stuff. +.if ${MK_SOURCELESS_UCODE} != "no" .if !(${COMPILER_TYPE} == "gcc" && ${COMPILER_VERSION} >= 50200) SUBDIR.${MK_USB}+= uathload .endif +.endif SUBDIR.${MK_USB}+= uhsoctl SUBDIR.${MK_USB}+= usbconfig SUBDIR.${MK_USB}+= usbdump -- 2.7.0 From 79fb234a406fc3a8885009355bee721dbdd968d2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 21:33:19 +0200 Subject: [PATCH 159/213] Detach fwcontrol from the build. It's pointless without firewire(4) --- usr.sbin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index b5e1961..2003b86 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -29,7 +29,6 @@ SUBDIR= adduser \ extattrctl \ fifolog \ fstyp \ - fwcontrol \ getfmac \ getpmac \ gstat \ -- 2.7.0 From b9bddbdfb1f85fac60ab0ed8c6b8695f0197c1b1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 23:58:46 +0200 Subject: [PATCH 160/213] sys/x86: Remove another FreeBSD references from a log message --- sys/x86/x86/mp_x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index 9594ff3..c8501a24 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -453,7 +453,7 @@ cpu_mp_announce(void) const char *hyperthread; int i; - printf("FreeBSD/SMP: %d package(s) x %d core(s)", + printf("SMP: %d package(s) x %d core(s)", mp_ncpus / (cpu_cores * cpu_logical), cpu_cores); if (hyperthreading_cpus > 1) printf(" x %d HTT threads", cpu_logical); -- 2.7.0 From 80e7bb12761dc1c1d33dc5883b5fcf8632b26f7a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 20 Aug 2015 10:12:27 +0200 Subject: [PATCH 161/213] pw(8): Increase minimal random password length to 32 --- usr.sbin/pw/pw_user.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.sbin/pw/pw_user.c b/usr.sbin/pw/pw_user.c index 30a2749..892347f 100644 --- a/usr.sbin/pw/pw_user.c +++ b/usr.sbin/pw/pw_user.c @@ -514,7 +514,7 @@ pw_password(struct userconf * cnf, char const * user, bool dryrun) switch (cnf->default_password) { case -1: /* Random password */ - l = (arc4random() % 8 + 8); /* 8 - 16 chars */ + l = 32 + (arc4random() % 8); /* 32 - 40 chars */ for (i = 0; i < l; i++) pwbuf[i] = chars[arc4random_uniform(sizeof(chars)-1)]; pwbuf[i] = '\0'; -- 2.7.0 From e33e9090fa874b6416fdeb36c8c1a5382abeecd6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 24 Aug 2015 14:31:05 +0200 Subject: [PATCH 162/213] Makefile.inc1: Respect WITHOUT_TCSH --- Makefile.inc1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 2e7cfd2..1b13293 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1592,8 +1592,13 @@ _gcc_tools= gnu/usr.bin/cc/cc_tools _rescue=rescue/rescue .endif +.if ${MK_TCSH} != "no" +_tcsh=bin/csh +.endif + +build-tools: .MAKE .for _tool in \ - bin/csh \ + ${_tcsh} \ bin/sh \ ${LOCAL_TOOL_DIRS} \ lib/ncurses/ncurses \ @@ -1749,7 +1754,7 @@ native-xtools: .PHONY bin/cat \ bin/chmod \ bin/cp \ - bin/csh \ + ${_tcsh} \ bin/echo \ bin/expr \ bin/hostname \ -- 2.7.0 From 6fdcfb56385c58eb2ee8ad83d448f171a7d5dbe8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 27 Aug 2015 17:10:31 +0200 Subject: [PATCH 163/213] etc/rc.d: Detach a couple of rc scripts that aren't relevant for ElectroBSD --- etc/rc.d/Makefile | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 72828d7..6f2f304 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -12,12 +12,8 @@ FILES= DAEMON \ addswap \ adjkerntz \ archdep \ - atm1 \ - atm2 \ - atm3 \ auditd \ auditdistd \ - bgfsck \ ${_bluetooth} \ bridge \ ${_bthidd} \ @@ -33,7 +29,6 @@ FILES= DAEMON \ dhclient \ dmesg \ dumpon \ - fsck \ gbde \ geli \ geli2 \ @@ -46,7 +41,6 @@ FILES= DAEMON \ hostname \ iovctl \ ip6addrctl \ - ipfilter \ ipfs \ ipfw \ ipmon \ @@ -72,7 +66,6 @@ FILES= DAEMON \ mountd \ mroute6d \ msgs \ - natd \ netif \ netoptions \ netwait \ @@ -89,15 +82,12 @@ FILES= DAEMON \ pf \ pflog \ pfsync \ - ppp \ - pppoed \ pwcheck \ quota \ random \ rarpd \ rctl \ resolv \ - rfcomm_pppd_server \ root \ route6d \ routing \ @@ -109,7 +99,6 @@ FILES= DAEMON \ securelevel \ serial \ soft-protection \ - sppp \ ${_sshd} \ statd \ static_arp \ -- 2.7.0 From f787e3888e7afe2a97d47c75bd1c858dd1969f61 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 9 Jul 2015 14:55:50 +0200 Subject: [PATCH 164/213] etc/pkg: Add and install ElectroBSD.conf and a pubkey file ... for testing purposes. While at it, detach FreeBSD.conf and the corresponding fingerprint file from the build. --- etc/pkg/ElectroBSD.conf | 21 +++++++++++++++++++++ etc/pkg/Makefile | 2 +- etc/pkg/electrobsd-test-repo.pub | 14 ++++++++++++++ share/keys/pkg/trusted/Makefile | 2 +- share/keys/pkg/trusted/electrobsd-test-key.conf | 2 ++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 etc/pkg/ElectroBSD.conf create mode 100644 etc/pkg/electrobsd-test-repo.pub create mode 100644 share/keys/pkg/trusted/electrobsd-test-key.conf diff --git a/etc/pkg/ElectroBSD.conf b/etc/pkg/ElectroBSD.conf new file mode 100644 index 0000000..4bf82d9 --- /dev/null +++ b/etc/pkg/ElectroBSD.conf @@ -0,0 +1,21 @@ +ElectroBSD: { + # Note that accessing this repository requires a local proxy + # that port forwards the incomming http requests to an authorized + # Tor client that forwards them to: http://gkpssb7hd77qznoa.onion/ + # + # To get the required secret you have to ask someone who knows it and + # come up with a good reason why you want to run unreproducible packages + # instead of building your packages from source. + # + # Example bootstrapping setup: + # Your server: pkg -> sshd + # | + # Your client: --> ssh -> privoxy -> tor + # | + # Outside your control: --> Tor network -> Tor HS + url: "http://127.0.0.1:8000/packages/${ABI}/", + mirror_type: "NONE", + signature_type: "pubkey", + pubkey: "/etc/pkg/electrobsd-test-repo.pub", + enabled: yes +} diff --git a/etc/pkg/Makefile b/etc/pkg/Makefile index abc8c64..614989f 100644 --- a/etc/pkg/Makefile +++ b/etc/pkg/Makefile @@ -2,7 +2,7 @@ NO_OBJ= -FILES= FreeBSD.conf +FILES= ElectroBSD.conf electrobsd-test-repo.pub FILESDIR= /etc/pkg FILESMODE= 644 diff --git a/etc/pkg/electrobsd-test-repo.pub b/etc/pkg/electrobsd-test-repo.pub new file mode 100644 index 0000000..ee213cb --- /dev/null +++ b/etc/pkg/electrobsd-test-repo.pub @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtwIgcSgQT2QJvxwDiKhh +xYidaxWaJL/EHXnYH8TpiujNeg8yZUCiT07MChw/wO3KrVY3SlhFkl8MHnwaFWNS +jSJFNQIHoVo8of0juDXhIMMY0DTVFVpSfgq+Ea0SrVZawQchcc7XODjp7mQOH0Nz +Z0FUpQ4z7TsdrPV/Kj/SQteK7Q0qAnDGT5GBxb+4m5qmNbv74GVpWmxJ5kC4OY3g +v8oTRZ5wG/mf5nKG+QXGKbcfoK+fd+NL9gp5awhvnp5cRo2+ZrQvnYKy5ym9O+jM +9hv5bL7OYIW7gErPdQd0+SFejcDg4TamY824krkjgIf7pnRVAuuSHiy8PaHmmNMd +StFo8IW6687C0X4gNuaPtR1TCkVZugrgev/w5GPp765HmE4WvrtPU4u9/EvWO2L6 +2bHmsJqNIGyq1lfHk/cx49CEhDPjRcYin8MKKzRFUR7CWiNDBz2GK6Lcfd78cG5I +HGJAthhi3rOOnLsTMuIQ/+GkVCDNwweLxSoLQUzGnheHqBjYyZSfZ8u9I/OKtwp3 +OMhGOD1/yuRaat+QER5VwiDP3AxcIlq6aNNytJW6l/kZZLVsr99YmROyYsTps0Zm +OXqxQFsZ6zzlYO95HP4ITJ7ubk84YPTvxbQ6LqsijqRI322aNvxxeEWyFxXpyCpp +uq50HtVoVX1a4ONsh/9l75sCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/share/keys/pkg/trusted/Makefile b/share/keys/pkg/trusted/Makefile index e55977c..f55afc9 100644 --- a/share/keys/pkg/trusted/Makefile +++ b/share/keys/pkg/trusted/Makefile @@ -1,6 +1,6 @@ # $FreeBSD$ -FILES= pkg.freebsd.org.2013102301 +FILES= electrobsd-test-key.conf FILESDIR= ${SHAREDIR}/keys/pkg/trusted FILESMODE= 644 diff --git a/share/keys/pkg/trusted/electrobsd-test-key.conf b/share/keys/pkg/trusted/electrobsd-test-key.conf new file mode 100644 index 0000000..e81a919 --- /dev/null +++ b/share/keys/pkg/trusted/electrobsd-test-key.conf @@ -0,0 +1,2 @@ +function: "sha256" +fingerprint: "8b1ef90f16ccc99342fb204f7ff57cdd31449e5ce1baa10c0b1b09593c44219d" -- 2.7.0 From 4ac4762c11fc7138f71a143b46f808b1eedb6a4e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 5 Sep 2015 22:46:09 +0200 Subject: [PATCH 165/213] dtrace: Add look-who-is-reaping It can be used to monitor and finetune the ARC cache reaper. The stack trace obviously isn't particular useful anymore now that the reaper is called from a single location. Obtained from: ElectroBSD --- share/dtrace/look-who-is-reaping | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 share/dtrace/look-who-is-reaping diff --git a/share/dtrace/look-who-is-reaping b/share/dtrace/look-who-is-reaping new file mode 100755 index 0000000..0db6324 --- /dev/null +++ b/share/dtrace/look-who-is-reaping @@ -0,0 +1,58 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * look-who-is-reaping + * + * Collect stack traces for reap_arc_caches(). + * + * Copyright (c) 2015 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + printf("%Y: Look who's (ARC cache) reaping .... Hit CTRL-C to exit.\n", walltimestamp); +} + +fbt::reap_arc_caches:entry +{ + @reapers[probefunc,stack(3)] = count(); + @reapers_total[probefunc,stack(3)] = count(); + reaped++ +} + +tick-10s +/reaped == 0/ +{ + printf("%Y: The ARC cache reaper has not been seen for 10 seconds!\n", walltimestamp); +} + +tick-10s +/reaped/ +{ + printf("%Y: The ARC reaper was called:\n", walltimestamp); + printa(@reapers); + trunc(@reapers); + reaped = 0 +} + +END +/reaped/ +{ + printf("%Y: The ARC reaper stats total:\n", walltimestamp); + printa(@reapers_total); +} -- 2.7.0 From d4afac1d9a3dedea3833c2e1ccaf00c3ed10a3b6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 17 Jul 2015 13:11:39 +0200 Subject: [PATCH 166/213] Let the ZFS ARC behave better under memory pressure Original author: Karl Denninger Source: https://bz-attachments.freebsd.org/attachment.cgi?id=152852&action=diff&collapsed=&context=patch&format=raw&headers=1 PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594 Commit adjusted to compile on i386 and to deal with merge conflicts with (at least): r263620/6fcf6199a: "Rename global cnt to vm_cnt to avoid shadowing" r286625/78648874e: "5376 arc_kmem_reap_now() should not result in clearing arc_no_grow" r286763/71fb6300f: "5497 lock contention on arcs_mtx" No functional change intended. --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 83 +++++++++++++++++----- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 47 ++++++++++-- .../opensolaris/uts/common/fs/zfs/dsl_pool.c | 7 +- .../opensolaris/uts/common/fs/zfs/sys/dsl_pool.h | 1 + .../contrib/opensolaris/uts/common/fs/zfs/zio.c | 4 +- 5 files changed, 116 insertions(+), 26 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index fad9874..2b11355 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -216,6 +216,15 @@ static int arc_dead; extern boolean_t zfs_prefetch_disable; /* + * KD 2015-02-10 + * We have to be able to test for UIO use inside the arc allocator. + * NOTE: DO NOT MODIFY HERE! + */ +extern int zio_use_uma; +extern int zfs_dynamic_write_buffer; + + +/* * The arc has filled available memory and has now warmed up. */ static boolean_t arc_warm; @@ -242,7 +251,7 @@ static void arc_free_target_init(void *unused __unused) { - zfs_arc_free_target = vm_pageout_wakeup_thresh; + zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -261,7 +270,9 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN, SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW, &arc_shrink_shift, 0, "log2(fraction of arc to reclaim)"); - +SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, + &zfs_dynamic_write_buffer, 0, + "Dynamically restrict dirty data when memory is low"); /* * We don't have a tunable for arc_free_target due to the dependency on * pagedaemon initialisation. @@ -3536,13 +3547,32 @@ extern kmem_cache_t *zio_buf_cache[]; extern kmem_cache_t *zio_data_buf_cache[]; extern kmem_cache_t *range_seg_cache; -static __noinline void -arc_kmem_reap_now(void) +static void __used +reap_arc_caches() { size_t i; kmem_cache_t *prev_cache = NULL; kmem_cache_t *prev_data_cache = NULL; + for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { + if (zio_buf_cache[i] != prev_cache) { + prev_cache = zio_buf_cache[i]; + kmem_cache_reap_now(zio_buf_cache[i]); + } + if (zio_data_buf_cache[i] != prev_data_cache) { + prev_data_cache = zio_data_buf_cache[i]; + kmem_cache_reap_now(zio_data_buf_cache[i]); + } + } + kmem_cache_reap_now(buf_cache); + kmem_cache_reap_now(hdr_full_cache); + kmem_cache_reap_now(hdr_l2only_cache); + kmem_cache_reap_now(range_seg_cache); +} + +static __noinline void +arc_kmem_reap_now(void) +{ DTRACE_PROBE(arc__kmem_reap_start); #ifdef _KERNEL if (arc_meta_used >= arc_meta_limit) { @@ -3560,20 +3590,7 @@ arc_kmem_reap_now(void) #endif #endif - for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { - if (zio_buf_cache[i] != prev_cache) { - prev_cache = zio_buf_cache[i]; - kmem_cache_reap_now(zio_buf_cache[i]); - } - if (zio_data_buf_cache[i] != prev_data_cache) { - prev_data_cache = zio_data_buf_cache[i]; - kmem_cache_reap_now(zio_data_buf_cache[i]); - } - } - kmem_cache_reap_now(buf_cache); - kmem_cache_reap_now(hdr_full_cache); - kmem_cache_reap_now(hdr_l2only_cache); - kmem_cache_reap_now(range_seg_cache); + reap_arc_caches(); #ifdef illumos if (zio_arena != NULL) { @@ -3608,11 +3625,28 @@ arc_reclaim_thread(void *dummy __unused) { clock_t growtime = 0; callb_cpr_t cpr; + int autoreap = 0; CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG); mutex_enter(&arc_reclaim_lock); while (!arc_reclaim_thread_exit) { +#ifdef _KERNEL +/* KD 2015-02-10 + * Protect against UMA free memory bloat. We already do this on a low-memory + * basis in the allocator; it has to happen there rather than here due to + * response time considerations. Make the call here once every 10 passes as + * well; this reclaims unused UMA buffers every 10 seconds on an idle system + * and more frequently if the reclaim thread gets woken up by low RAM + * conditions. + */ + if ((zio_use_uma) && (autoreap++ == 10)) { + autoreap = 0; + DTRACE_PROBE(arc__reclaim_timed_reap); + reap_arc_caches(); + } +#endif /* _KERNEL */ + int64_t free_memory = arc_available_memory(); uint64_t evicted = 0; @@ -3878,6 +3912,19 @@ arc_get_data_buf(arc_buf_t *buf) arc_space_consume(size, ARC_SPACE_META); } else { ASSERT(type == ARC_BUFC_DATA); +#ifdef _KERNEL +/* KD 2015-02-10 + * It would be nice if we could leave this to the arc_reclaim thread. + * Unfortunately we cannot; the test has to be done here as well, because + * under heavy I/O demand we can grab enough RAM fast enough to induce + * nasty oscillation problems. Fortunately we only need to call this when + * the system is under reasonably-severe memory stress. + */ + if (zio_use_uma && (ptob(vm_cnt.v_free_count) + size < ptob(vm_cnt.v_free_target))) { + DTRACE_PROBE3(arc__alloc_lowmem_reap, int, vm_cnt.v_free_count, int, size, int, vm_cnt.v_free_target); + reap_arc_caches(); + } +#endif /* _KERNEL */ buf->b_data = zio_data_buf_alloc(size); arc_space_consume(size, ARC_SPACE_DATA); } diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 65a017f..2e7b4dc 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -42,6 +42,8 @@ typedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn, uint64_t arg1, uint64_t arg2); +extern int zio_use_uma; /* Needs to be visible; DO NOT MODIFY! */ +int zfs_dynamic_write_buffer = 1; /* Dynamically tune writes */ dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) @@ -1060,7 +1062,7 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) { dsl_pool_t *dp = tx->tx_pool; uint64_t delay_min_bytes = - zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100; + zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; hrtime_t wakeup, min_tx_time, now; if (dirty <= delay_min_bytes) @@ -1072,11 +1074,11 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) * have to handle the case of it being >= the max, which could * cause a divide-by-zero if it's == the max. */ - ASSERT3U(dirty, <, zfs_dirty_data_max); + ASSERT3U(dirty, <, zfs_dirty_data_max_internal); now = gethrtime(); min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max - dirty); + (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); if (now > tx->tx_start + min_tx_time) return; @@ -1281,6 +1283,7 @@ int dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) { int err; + static uint64_t last_max; ASSERT(tx->tx_txg == 0); ASSERT(txg_how == TXG_WAIT || txg_how == TXG_NOWAIT || @@ -1293,6 +1296,42 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) if (txg_how == TXG_WAITED) tx->tx_waited = B_TRUE; +#ifdef _KERNEL + /* + * KD 2014-09-22 + * If UMA is enabled it can only return a previously-used block + * of identical size to what it had out before. If it's not the + * same size it will allocate a new one. This is a problem because + * dirty_data_max is the total dirty write data allowed out at any + * given time, but with UMA on that can multiply by the number of + * different block sizes (!!) requested in terms of free RAM that + * is left allocated but unused. For this reason never allow + * dirty_data_max to exceed the difference between the paging + * threshold and the current free memory, with a minimum of 256MB. + * This throttles "burst" allocations and prevents the system from + * choking during times of high write I/O demand. + * + * We allow this to be turned off if you want with + * "vfs.zfs_dynamic_write_buffer=0", which can be done in real time. + * + * Note that we work on the zfs_dirty_data_max_internal variable, + * because the user may set zfs_dirty_data_max himself and we must + * must honor that as a hard cap so it remains a usable tunable value. + */ + if (zio_use_uma & zfs_dynamic_write_buffer) { + zfs_dirty_data_max_internal = 1 << 28; + zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max_max); + if (last_max != (zfs_dirty_data_max_internal / (1024 * 1024))) { + last_max = zfs_dirty_data_max_internal / (1024 * 1024); + DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); + } + } else { + zfs_dirty_data_max_internal = zfs_dirty_data_max; + } +#endif /* _KERNEL */ + while ((err = dmu_tx_try_assign(tx, txg_how)) != 0) { dmu_tx_unassign(tx); @@ -1323,7 +1362,7 @@ dmu_tx_wait(dmu_tx_t *tx) * space. */ mutex_enter(&dp->dp_lock); - while (dp->dp_dirty_total >= zfs_dirty_data_max) + while (dp->dp_dirty_total >= zfs_dirty_data_max_internal) cv_wait(&dp->dp_spaceavail_cv, &dp->dp_lock); uint64_t dirty = dp->dp_dirty_total; mutex_exit(&dp->dp_lock); diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index 189ca19..0ec4fd6 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -98,8 +98,11 @@ /* * zfs_dirty_data_max will be set to zfs_dirty_data_max_percent% of all memory, * capped at zfs_dirty_data_max_max. It can also be overridden in /etc/system. + * We also dynamically tune during low memory, honoring the sysctl set, so + * internal comparisons are against zfs_dirty_data_max_internal. */ uint64_t zfs_dirty_data_max; +uint64_t zfs_dirty_data_max_internal; uint64_t zfs_dirty_data_max_max = 4ULL * 1024 * 1024 * 1024; int zfs_dirty_data_max_percent = 10; @@ -553,7 +556,7 @@ dsl_pool_dirty_delta(dsl_pool_t *dp, int64_t delta) * Note: we signal even when increasing dp_dirty_total. * This ensures forward progress -- each thread wakes the next waiter. */ - if (dp->dp_dirty_total <= zfs_dirty_data_max) + if (dp->dp_dirty_total <= zfs_dirty_data_max_internal) cv_signal(&dp->dp_spaceavail_cv); } @@ -732,7 +735,7 @@ boolean_t dsl_pool_need_dirty_delay(dsl_pool_t *dp) { uint64_t delay_min_bytes = - zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100; + zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; boolean_t rv; mutex_enter(&dp->dp_lock); diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h index 0e27a53..ee97b57 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h @@ -50,6 +50,7 @@ struct dmu_tx; struct dsl_scan; extern uint64_t zfs_dirty_data_max; +extern uint64_t zfs_dirty_data_max_internal; extern uint64_t zfs_dirty_data_max_max; extern uint64_t zfs_dirty_data_sync; extern int zfs_dirty_data_max_percent; diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c index 901e618..089c314 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c @@ -44,9 +44,9 @@ SYSCTL_DECL(_vfs_zfs); SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); #if defined(__amd64__) -static int zio_use_uma = 1; +int zio_use_uma = 1; #else -static int zio_use_uma = 0; +int zio_use_uma = 0; #endif SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, use_uma, CTLFLAG_RDTUN, &zio_use_uma, 0, "Use uma(9) for ZIO allocations"); -- 2.7.0 From de88064069ab775ff802d45cff34e63b1b00c6e4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 11:33:46 +0200 Subject: [PATCH 167/213] ZFS ARC: Add a missing & to get a logical and instead of a binary one --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 2e7b4dc..6169a6d 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1318,7 +1318,7 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) * because the user may set zfs_dirty_data_max himself and we must * must honor that as a hard cap so it remains a usable tunable value. */ - if (zio_use_uma & zfs_dynamic_write_buffer) { + if (zio_use_uma && zfs_dynamic_write_buffer) { zfs_dirty_data_max_internal = 1 << 28; zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); -- 2.7.0 From e850e955995a6651b9d30c8abad42d3b26390d37 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 3 Sep 2015 13:58:29 +0200 Subject: [PATCH 168/213] ZFS ARC: Move the ARC cache reaping out of the hot path ... and use a time-based rate limiting approach that can be controlled with the vfs.zfs.arc_reap_delay_min sysctl. Previously memory pressure could result in several thousand ARC reaping attempts per second. This wasn't merely inefficient but also caused lock contention and poor latency for ZFS operations that relied on arc_get_data_buf(). For details see: https://www.fabiankeil.de/gehacktes/electrobsd/zfs-arc-tuning/ Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 97 ++++++++++++++-------- 1 file changed, 63 insertions(+), 34 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 2b11355..232be12 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -3547,8 +3547,21 @@ extern kmem_cache_t *zio_buf_cache[]; extern kmem_cache_t *zio_data_buf_cache[]; extern kmem_cache_t *range_seg_cache; -static void __used -reap_arc_caches() +/* + * Used by arc_kmem_reap_now() and consider_reaping_arc_caches() + * to limit the time spent reaping. + * + * The arc_reaping_in_progress is a (somewhat racy) left over from a + * previous version of this code which could trigger multiple ARC cache + * reapings in parallel which should be avoided to reduce lock + * contention. It's hasn't been removed yet to encourage further + * experimenting. + */ +static unsigned int arc_reaping_in_progress = 0; +static sbintime_t last_reaping = 0; + +static void __noinline +reap_arc_caches(void) { size_t i; kmem_cache_t *prev_cache = NULL; @@ -3574,6 +3587,8 @@ static __noinline void arc_kmem_reap_now(void) { DTRACE_PROBE(arc__kmem_reap_start); + arc_reaping_in_progress++; + #ifdef _KERNEL if (arc_meta_used >= arc_meta_limit) { /* @@ -3601,9 +3616,49 @@ arc_kmem_reap_now(void) vmem_qcache_reap(zio_arena); } #endif +#ifdef _KERNEL + last_reaping = getsbinuptime(); +#endif + arc_reaping_in_progress = 0; DTRACE_PROBE(arc__kmem_reap_end); } + +/* + * Declared writable to allow resetting it. + * XXX: Should probably be a uint64 and integrated with kstat. + */ +static unsigned int arc_cache_reapings_skipped = 0; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_skipped, CTLFLAG_RW, + &arc_cache_reapings_skipped, 0, "Number of times the ARC caches have not been reaped due to the reap delay"); + +static unsigned int min_arc_reap_delay = 200; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_reap_delay_min, CTLFLAG_RW, + &min_arc_reap_delay, 200, "Minimum delay between ARC cache reapings (milliseconds)"); + +static void __noinline +consider_reaping_arc_caches(void) +{ +#ifdef _KERNEL + sbintime_t now; + + if (arc_reaping_in_progress) { + /* Already reaping in another thread. */ + arc_cache_reapings_skipped++; + return; + } + + now = getsbinuptime(); + if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) + { + /* Too soon to reap again. */ + arc_cache_reapings_skipped++; + return; + } +#endif + arc_kmem_reap_now(); +} + /* * Threads can block in arc_get_data_buf() waiting for this thread to evict * enough data and signal them to proceed. When this happens, the threads in @@ -3625,28 +3680,11 @@ arc_reclaim_thread(void *dummy __unused) { clock_t growtime = 0; callb_cpr_t cpr; - int autoreap = 0; CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG); mutex_enter(&arc_reclaim_lock); while (!arc_reclaim_thread_exit) { -#ifdef _KERNEL -/* KD 2015-02-10 - * Protect against UMA free memory bloat. We already do this on a low-memory - * basis in the allocator; it has to happen there rather than here due to - * response time considerations. Make the call here once every 10 passes as - * well; this reclaims unused UMA buffers every 10 seconds on an idle system - * and more frequently if the reclaim thread gets woken up by low RAM - * conditions. - */ - if ((zio_use_uma) && (autoreap++ == 10)) { - autoreap = 0; - DTRACE_PROBE(arc__reclaim_timed_reap); - reap_arc_caches(); - } -#endif /* _KERNEL */ - int64_t free_memory = arc_available_memory(); uint64_t evicted = 0; @@ -3663,8 +3701,6 @@ arc_reclaim_thread(void *dummy __unused) */ growtime = ddi_get_lbolt() + (arc_grow_retry * hz); - arc_kmem_reap_now(); - /* * If we are still low on memory, shrink the ARC * so that we have arc_shrink_min free space. @@ -3738,6 +3774,12 @@ arc_user_evicts_thread(void *dummy __unused) while (!arc_user_evicts_thread_exit) { mutex_exit(&arc_user_evicts_lock); + /* + * Consider reaping the ARC caches at least once per + * second, but more often when signalled under pressure. + */ + consider_reaping_arc_caches(); + arc_do_user_evicts(); /* @@ -3912,19 +3954,6 @@ arc_get_data_buf(arc_buf_t *buf) arc_space_consume(size, ARC_SPACE_META); } else { ASSERT(type == ARC_BUFC_DATA); -#ifdef _KERNEL -/* KD 2015-02-10 - * It would be nice if we could leave this to the arc_reclaim thread. - * Unfortunately we cannot; the test has to be done here as well, because - * under heavy I/O demand we can grab enough RAM fast enough to induce - * nasty oscillation problems. Fortunately we only need to call this when - * the system is under reasonably-severe memory stress. - */ - if (zio_use_uma && (ptob(vm_cnt.v_free_count) + size < ptob(vm_cnt.v_free_target))) { - DTRACE_PROBE3(arc__alloc_lowmem_reap, int, vm_cnt.v_free_count, int, size, int, vm_cnt.v_free_target); - reap_arc_caches(); - } -#endif /* _KERNEL */ buf->b_data = zio_data_buf_alloc(size); arc_space_consume(size, ARC_SPACE_DATA); } -- 2.7.0 From 0d70a72abcfcda406f22c419d01d8ca943429aa6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Oct 2015 13:12:26 +0200 Subject: [PATCH 169/213] ZFS ARC: Ignore the reap delay when under memory pressure --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 232be12..b41153f 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -3631,6 +3631,9 @@ arc_kmem_reap_now(void) static unsigned int arc_cache_reapings_skipped = 0; SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_skipped, CTLFLAG_RW, &arc_cache_reapings_skipped, 0, "Number of times the ARC caches have not been reaped due to the reap delay"); +static unsigned int arc_cache_reapings_forced = 0; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_forced, CTLFLAG_RW, + &arc_cache_reapings_forced, 0, "Number of times the ARC caches reap delay was ignored due to memory pressure"); static unsigned int min_arc_reap_delay = 200; SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_reap_delay_min, CTLFLAG_RW, @@ -3649,11 +3652,17 @@ consider_reaping_arc_caches(void) } now = getsbinuptime(); - if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) - { - /* Too soon to reap again. */ - arc_cache_reapings_skipped++; - return; + if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) { + /* + * Skip the reaping unless there's memory pressure + * in which case we would risk vm deadlocks. We don't + * use vm_page_count_severe() as it triggers too late. + */ + if (!vm_page_count_min()) { + arc_cache_reapings_skipped++; + return; + } + arc_cache_reapings_forced++; } #endif arc_kmem_reap_now(); -- 2.7.0 From 66bc49506d98b95e37de6e8c44ebf4e93b9a37aa Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Sep 2015 20:07:22 +0200 Subject: [PATCH 170/213] dtrace: Import fbt-time which llquantizes the time spent in a given kernel function Among other things this can be useful for general system tuning and analysis of lock contention (once you know where to expect it). Obtained from: ElectroBSD --- share/dtrace/fbt-time | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 share/dtrace/fbt-time diff --git a/share/dtrace/fbt-time b/share/dtrace/fbt-time new file mode 100755 index 0000000..27a950b --- /dev/null +++ b/share/dtrace/fbt-time @@ -0,0 +1,80 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * fbt-time + * + * Measures the time spent in a given kernel function and llquantizes + * the results for the traced calls. Example output can be seen at: + * http://www.fabiankeil.de/gehacktes/electrobsd/zfs-arc-tuning/ + * + * WARNING: Tracing kernel functions that get executed several thousand + * times per second may negatively impact system performance + * and can even cause brief lockups. Before using this script + * on production systems you may want to sample the traced + * kernel function first. + * + * XXX: This script should be fleshed out to be more resilient to + * incorrect input and provide more awesome stats. A better name + * would be nice, too. + * + * Copyright (c) 2014 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + traced_function = $$1; + + multiple = "microseconds"; + divisor = 1000; + executions = 0; + + this->timestamp = walltimestamp; + this->msecs = (this->timestamp / 1000000) % 1000; + printf("%Y.%.3d: Tracing '%s'. Hit CTRL-C to exit.\n", + this->timestamp, this->msecs, traced_function); +} + +fbt::$$1:entry +{ + self->timestamp = timestamp; + executions++; +} + +fbt::$$1:return +/self->timestamp/ +{ + this->elapsed = (timestamp - self->timestamp) / divisor; + @elapsed_time = llquantize(this->elapsed, 10, 0, 5, 10); + @elapsed_avg["Average"] = avg(this->elapsed); + self->elapsed = 0; +} + +tick-10sec, +END +/executions/ +{ + printf("%Y: Runtime for %s() in %s\n", walltimestamp, traced_function, multiple); + printa(@elapsed_avg); + printa(@elapsed_time); +} + +END +/executions == 0/ +{ + printf("%Y: Looks like %s() wasn't called yet.\n", walltimestamp, traced_function); +} -- 2.7.0 From 40892bebd418eee5c7bd239548b69d2388e4359d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Mar 2015 13:06:05 +0100 Subject: [PATCH 171/213] ifconfig: Implement MAC address randomization Usage: ifconfig $nic ether random Inspired by the OpenBSD code which unfortunately can't be imported directly due to code differences. --- sbin/ifconfig/af_link.c | 8 ++++++++ sbin/ifconfig/ifconfig.8 | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sbin/ifconfig/af_link.c b/sbin/ifconfig/af_link.c index 4a4b661..984af23 100644 --- a/sbin/ifconfig/af_link.c +++ b/sbin/ifconfig/af_link.c @@ -81,6 +81,14 @@ link_getaddr(const char *addr, int which) if (which != ADDR) errx(1, "can't set link-level netmask or broadcast"); + if (!strcmp(addr, "random")) { + sa->sa_family = AF_LINK; + sa->sa_len = ETHER_ADDR_LEN; + arc4random_buf(&sa->sa_data, sa->sa_len); + /* Make sure it's a non-multicast hardware address */ + sa->sa_data[0] &= 0xfc; + return; + } if ((temp = malloc(strlen(addr) + 2)) == NULL) errx(1, "malloc failed"); temp[0] = ':'; diff --git a/sbin/ifconfig/ifconfig.8 b/sbin/ifconfig/ifconfig.8 index a5d7380..50f524b 100644 --- a/sbin/ifconfig/ifconfig.8 +++ b/sbin/ifconfig/ifconfig.8 @@ -140,7 +140,10 @@ parameter below for more information. The link-level .Pq Dq link address -is specified as a series of colon-separated hex digits. +is specified as a series of colon-separated hex digits +or, if the address is +.Dq random , +will be chosen randomly. This can be used to, for example, set a new MAC address on an Ethernet interface, though the mechanism used is not Ethernet specific. -- 2.7.0 From afbae100594a426160454e4a54591d141eafd412 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Mar 2015 14:22:43 +0100 Subject: [PATCH 172/213] Import rc.d/rether which randomizes MAC addresses ... provided ifconfig supports "either random". --- etc/rc.d/Makefile | 1 + etc/rc.d/rether | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100755 etc/rc.d/rether diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 6f2f304..6c90ff2 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -88,6 +88,7 @@ FILES= DAEMON \ rarpd \ rctl \ resolv \ + rether \ root \ route6d \ routing \ diff --git a/etc/rc.d/rether b/etc/rc.d/rether new file mode 100755 index 0000000..51e8603 --- /dev/null +++ b/etc/rc.d/rether @@ -0,0 +1,97 @@ +#!/bin/sh +# +########################################################################### +# +# rether - Randomizes MAC addresses +# +# Add the following line to /etc/rc.conf to randomize the MAC +# address for all recognized network interfaces that got one +# at startup: +# +# rether_enable="YES" +# +# You can specify the interfaces manually like this: +# +# rether_interfaces="bge0 iwn0" +# +# By default rether runs before netif so cloned devices aren't +# expected to exist yet. In case of wlan clones the MAC address +# of the parent is inherited so as long as it's randomized the +# clone should be fine too. +# +# Note that MAC address mismatches between clone and parent may +# prevent the clone from working as expected. If you intend to +# run this script after the system is up, you may want to +# explicitly set rether_interfaces to skip clones. +# +# Rether requires an ifconfig version that understands "ether random". +# +########################################################################### +# +# Copyright (c) 2014 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# PROVIDE: rether +# REQUIRE: FILESYSTEMS +# BEFORE: netif + +. /etc/rc.subr + +name="rether" +rcvar="rether_enable" + +rether_enable="${rether_enable-NO}" + +start_cmd="rether_start" +stop_cmd=":" + +# For the reason given above, it may make sense to skip cloned devices. +get_ethernet_interfaces() { + ifconfig -l ether +} + +get_ethernet_address() { + local interface="${1}" + + ifconfig "${interface}" ether | awk '/ether/ {print $2}' +} + +rether_start() { + local \ + interface \ + ethernet_address + + if [ -z "${rether_interfaces}" ]; then + rether_interfaces="$(get_ethernet_interfaces)" + fi + + echo "Randomizing MAC addresses for: ${rether_interfaces}" + + for interface in $rether_interfaces; + do + ethernet_address="$(get_ethernet_address "${interface}")" + + ifconfig "${interface}" ether random + + if [ "${ethernet_address}" = "$(get_ethernet_address "${interface}")" ]; then + echo "Failed to randomize MAC address for ${interface}: ${ethernet_address}" + fi + done +} + +load_rc_config "${name}" +run_rc_command "${1}" -- 2.7.0 From 8161244e0eaead684992a08ffdd0e124ca20dfc4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Sep 2015 17:20:31 +0200 Subject: [PATCH 173/213] share/mk/src.opts.mk: Detach LIB32 from the build --- share/mk/src.opts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index ada687f..71b480b 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -103,7 +103,6 @@ __DEFAULT_YES_OPTIONS = \ LDNS \ LDNS_UTILS \ LEGACY_CONSOLE \ - LIB32 \ LIBPTHREAD \ LIBTHR \ LOCALES \ @@ -178,6 +177,7 @@ __DEFAULT_NO_OPTIONS += \ GCOV \ IPFILTER \ ISCSI \ + LIB32 \ NDIS \ RBOOTD \ PC_SYSINSTALL \ -- 2.7.0 From ecb48b34196bbc768e97128282c547e6729688f9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 15:19:51 +0200 Subject: [PATCH 174/213] Prevent inlining of vm_pageout_scan() to allow fbt tracing --- sys/vm/vm_pageout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index b7f6887..d38e985 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -874,7 +874,7 @@ unlock_mp: * pass 1 - Move inactive to cache or free * pass 2 - Launder dirty pages */ -static void +static void __noinline vm_pageout_scan(struct vm_domain *vmd, int pass) { vm_page_t m, next; -- 2.7.0 From 57f86addb2f0d38f826da0504ebe4dd45cc4e7b6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 15:33:12 +0200 Subject: [PATCH 175/213] Attempt to fix MAC address randomization after r287197 --- etc/Makefile | 1 + etc/mac-randomization.subr | 76 ++++++++++++++++++++++++++++++++++++ etc/rc.d/Makefile | 1 - etc/rc.d/netif | 3 ++ etc/rc.d/rether | 97 ---------------------------------------------- 5 files changed, 80 insertions(+), 98 deletions(-) create mode 100755 etc/mac-randomization.subr delete mode 100755 etc/rc.d/rether diff --git a/etc/Makefile b/etc/Makefile index 27d1ba4..0efbf51 100644 --- a/etc/Makefile +++ b/etc/Makefile @@ -27,6 +27,7 @@ BIN1= crontab \ login.access \ login.conf \ mac.conf \ + mac-randomization.subr \ motd \ netconfig \ network.subr \ diff --git a/etc/mac-randomization.subr b/etc/mac-randomization.subr new file mode 100755 index 0000000..d486bf4 --- /dev/null +++ b/etc/mac-randomization.subr @@ -0,0 +1,76 @@ +#!/bin/sh +# +########################################################################### +# +# Sub routines to randomizes MAC addresses after r287197 (WIP!) +# +# Add the following line to /etc/rc.conf to randomize the MAC +# address for all recognized network interfaces that got one +# at startup: +# +# rether_enable="YES" +# +# You can specify the interfaces manually like this: +# +# rether_interfaces="bge0 iwn0" +# +# Rether requires an ifconfig version that understands "ether random". +# +########################################################################### +# +# Copyright (c) 2014 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +rether_enable="${rether_enable-NO}" + +get_ethernet_interfaces() { + ifconfig -l ether +} + +get_ethernet_address() { + local interface="${1}" + + ifconfig "${interface}" ether | awk '/ether/ {print $2}' +} + +randomize_mac_addresses() { + local \ + interface \ + ethernet_address + + if [ "${rether_enable}" = "NO" ]; then + echo "Not randomizing any MAC addresses!" + return 0 + fi + + if [ -z "${rether_interfaces}" ]; then + rether_interfaces="$(get_ethernet_interfaces)" + fi + + echo "Randomizing MAC addresses for: ${rether_interfaces}" + + for interface in $rether_interfaces; + do + ethernet_address="$(get_ethernet_address "${interface}")" + + ifconfig "${interface}" ether random + + if [ "${ethernet_address}" = "$(get_ethernet_address "${interface}")" ]; then + echo "Failed to randomize MAC address for ${interface}: ${ethernet_address}" + fi + done +} diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 6c90ff2..6f2f304 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -88,7 +88,6 @@ FILES= DAEMON \ rarpd \ rctl \ resolv \ - rether \ root \ route6d \ routing \ diff --git a/etc/rc.d/netif b/etc/rc.d/netif index 5ed9562..2499a27 100755 --- a/etc/rc.d/netif +++ b/etc/rc.d/netif @@ -32,6 +32,7 @@ . /etc/rc.subr . /etc/network.subr +. /etc/mac-randomization.subr name="netif" rcvar="${name}_enable" @@ -73,6 +74,8 @@ netif_start() # Create cloned interfaces clone_up $cmdifn + randomize_mac_addresses + # Rename interfaces. ifnet_rename $cmdifn diff --git a/etc/rc.d/rether b/etc/rc.d/rether deleted file mode 100755 index 51e8603..0000000 --- a/etc/rc.d/rether +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/sh -# -########################################################################### -# -# rether - Randomizes MAC addresses -# -# Add the following line to /etc/rc.conf to randomize the MAC -# address for all recognized network interfaces that got one -# at startup: -# -# rether_enable="YES" -# -# You can specify the interfaces manually like this: -# -# rether_interfaces="bge0 iwn0" -# -# By default rether runs before netif so cloned devices aren't -# expected to exist yet. In case of wlan clones the MAC address -# of the parent is inherited so as long as it's randomized the -# clone should be fine too. -# -# Note that MAC address mismatches between clone and parent may -# prevent the clone from working as expected. If you intend to -# run this script after the system is up, you may want to -# explicitly set rether_interfaces to skip clones. -# -# Rether requires an ifconfig version that understands "ether random". -# -########################################################################### -# -# Copyright (c) 2014 Fabian Keil -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR -# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT -# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. -########################################################################### - -# PROVIDE: rether -# REQUIRE: FILESYSTEMS -# BEFORE: netif - -. /etc/rc.subr - -name="rether" -rcvar="rether_enable" - -rether_enable="${rether_enable-NO}" - -start_cmd="rether_start" -stop_cmd=":" - -# For the reason given above, it may make sense to skip cloned devices. -get_ethernet_interfaces() { - ifconfig -l ether -} - -get_ethernet_address() { - local interface="${1}" - - ifconfig "${interface}" ether | awk '/ether/ {print $2}' -} - -rether_start() { - local \ - interface \ - ethernet_address - - if [ -z "${rether_interfaces}" ]; then - rether_interfaces="$(get_ethernet_interfaces)" - fi - - echo "Randomizing MAC addresses for: ${rether_interfaces}" - - for interface in $rether_interfaces; - do - ethernet_address="$(get_ethernet_address "${interface}")" - - ifconfig "${interface}" ether random - - if [ "${ethernet_address}" = "$(get_ethernet_address "${interface}")" ]; then - echo "Failed to randomize MAC address for ${interface}: ${ethernet_address}" - fi - done -} - -load_rc_config "${name}" -run_rc_command "${1}" -- 2.7.0 From afa8b12eb958a1ec54e35e6ad94b80b56cc1d091 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 09:53:58 +0200 Subject: [PATCH 176/213] sys/boot/i386: Detach libfirewire from the build --- sys/boot/i386/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/i386/Makefile b/sys/boot/i386/Makefile index d812d54..4b7ca27 100644 --- a/sys/boot/i386/Makefile +++ b/sys/boot/i386/Makefile @@ -3,7 +3,7 @@ .include SUBDIR= mbr pmbr boot0 boot0sio btx boot2 cdboot gptboot \ - libi386 libfirewire loader + libi386 loader # special boot programs, 'self-extracting boot2+loader' SUBDIR+= pxeldr -- 2.7.0 From 049c690daa8bfe0e4c96b897cbc987e1785c39bf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 29 Sep 2015 22:31:45 +0200 Subject: [PATCH 177/213] README: Spell out GPL correctly, the G does not stand for GNU --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index 266dfb1..a5d13c7 100644 --- a/README +++ b/README @@ -47,8 +47,8 @@ etc Template files for /etc. games Amusements. -gnu Various commands and libraries under the GNU Public License. - Please see gnu/COPYING* for more information. +gnu Various commands and libraries under the GNU General Public + License. Please see gnu/COPYING* for more information. include System include files. -- 2.7.0 From 136e6c907f4132d31cbd8020047e1168205635f3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 30 Sep 2015 21:15:20 +0200 Subject: [PATCH 178/213] Add two DTrace scripts. WIP --- share/dtrace/io-breakdown | 19 +++++++++++++++++++ share/dtrace/open-ios | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100755 share/dtrace/io-breakdown create mode 100755 share/dtrace/open-ios diff --git a/share/dtrace/io-breakdown b/share/dtrace/io-breakdown new file mode 100755 index 0000000..ee9e592 --- /dev/null +++ b/share/dtrace/io-breakdown @@ -0,0 +1,19 @@ +#!/usr/sbin/dtrace -s + +/* This is the example from dtrace_io(4) */ + +#pragma D option quiet +#pragma D option dynvarsize=16m + +#pragma D option quiet + +io:::start +{ + @[args[1]->device_name, execname, pid] = sum(args[0]->bio_bcount); +} + +END +{ + printf("%10s %20s %10s %15s", "DEVICE", "APP", "PID", "BYTES"); + printa("%10s %20s %10d %15@d", @); +} diff --git a/share/dtrace/open-ios b/share/dtrace/open-ios new file mode 100755 index 0000000..ee8bbe0 --- /dev/null +++ b/share/dtrace/open-ios @@ -0,0 +1,31 @@ +#!/usr/sbin/dtrace -s + +#pragma D option quiet +#pragma D option dynvarsize=16m + +dtrace:::BEGIN +{ + printf("Tracing... Hit Ctrl-C to end.\n"); +} + +io:::start +{ + printf("%Y: Device name\n", walltimestamp); + print(args[1]->device_name); + start_time[arg0] = timestamp; + @open_ios = count(); +} + +io:::done +/this->start = start_time[arg0]/ +{ + @done_ios = count(); +} + +tick-10s +{ + printf("%Y: Started ios", walltimestamp); + printa(@open_ios); + printf("%Y: Done ios", walltimestamp); + printa(@done_ios); +} -- 2.7.0 From 621c469016e52dbb1290f133e6efd9da6f3c6e8b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 13:04:33 +0200 Subject: [PATCH 179/213] games/fortune/datfiles: Do not install non-free gerrold.limerick --- usr.bin/fortune/datfiles/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.bin/fortune/datfiles/Makefile b/usr.bin/fortune/datfiles/Makefile index 1eabaa4..5d979d9 100644 --- a/usr.bin/fortune/datfiles/Makefile +++ b/usr.bin/fortune/datfiles/Makefile @@ -5,7 +5,7 @@ DB= fortunes freebsd-tips murphy startrek zippy # TO AVOID INSTALLING THE POTENTIALLY OFFENSIVE FORTUNES, COMMENT OUT THE # NEXT LINE. -DB+= limerick murphy-o gerrold.limerick +DB+= limerick murphy-o BLDS= ${DB:S/$/.dat/} FILES= ${DB} ${BLDS} -- 2.7.0 From 79fc29c31ef53ac9f89c9e9ee61fb3d2d66c3c71 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 12:53:54 +0200 Subject: [PATCH 180/213] fortune/datafiles: Add German fundamental rights (poorly formatted) Enforcing these rights is left as an excercise for the reader. --- usr.bin/fortune/datfiles/Makefile | 2 +- usr.bin/fortune/datfiles/grundrechte | 346 +++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 usr.bin/fortune/datfiles/grundrechte diff --git a/usr.bin/fortune/datfiles/Makefile b/usr.bin/fortune/datfiles/Makefile index 5d979d9..0f25ce0 100644 --- a/usr.bin/fortune/datfiles/Makefile +++ b/usr.bin/fortune/datfiles/Makefile @@ -1,7 +1,7 @@ # @(#)Makefile 8.2 (Berkeley) 4/19/94 # $FreeBSD$ -DB= fortunes freebsd-tips murphy startrek zippy +DB= grundrechte fortunes freebsd-tips murphy startrek zippy # TO AVOID INSTALLING THE POTENTIALLY OFFENSIVE FORTUNES, COMMENT OUT THE # NEXT LINE. diff --git a/usr.bin/fortune/datfiles/grundrechte b/usr.bin/fortune/datfiles/grundrechte new file mode 100644 index 0000000..c9a0988 --- /dev/null +++ b/usr.bin/fortune/datfiles/grundrechte @@ -0,0 +1,346 @@ +% +Grundgesetz Artikel 1 +(1) Die Würde des Menschen ist unantastbar. Sie zu achten und zu +schützen ist Verpflichtung aller staatlichen Gewalt. +(2) Das Deutsche Volk bekennt sich darum zu unverletzlichen und +unveräußerlichen Menschenrechten als Grundlage jeder menschlichen +Gemeinschaft, des Friedens und der Gerechtigkeit in der Welt. +(3) Die nachfolgenden Grundrechte binden Gesetzgebung, vollziehende +Gewalt und Rechtsprechung als unmittelbar geltendes Recht. + % +Grundgesetz Artikel 2 +(1) Jeder hat das Recht auf die freie Entfaltung seiner +Persönlichkeit, soweit er nicht die Rechte anderer verletzt und nicht +gegen die verfassungsmäßige Ordnung oder das Sittengesetz verstößt. +(2) Jeder hat das Recht auf Leben und körperliche Unversehrtheit. Die +Freiheit der Person ist unverletzlich. In diese Rechte darf nur auf +Grund eines Gesetzes eingegriffen werden. +% +Grundgesetz Artikel 3 +(1) Alle Menschen sind vor dem Gesetz gleich. +(2) Männer und Frauen sind gleichberechtigt. Der Staat fördert die +tatsächliche Durchsetzung der Gleichberechtigung von Frauen und +Männern und wirkt auf die Beseitigung bestehender Nachteile hin. +(3) Niemand darf wegen seines Geschlechtes, seiner Abstammung, seiner +Rasse, seiner Sprache, seiner Heimat und Herkunft, seines Glaubens, +seiner religiösen oder politischen Anschauungen benachteiligt oder +bevorzugt werden. Niemand darf wegen seiner Behinderung benachteiligt +werden. +% +Grundgesetz Artikel 4 +(1) Die Freiheit des Glaubens, des Gewissens und die Freiheit des +religiösen und weltanschaulichen Bekenntnisses sind unverletzlich. +(2) Die ungestörte Religionsausübung wird gewährleistet. +(3) Niemand darf gegen sein Gewissen zum Kriegsdienst mit der Waffe +gezwungen werden. Das Nähere regelt ein Bundesgesetz. +% +Grundgesetz Artikel 5 +(1) Jeder hat das Recht, seine Meinung in Wort, Schrift und Bild frei +zu äußern und zu verbreiten und sich aus allgemein zugänglichen +Quellen ungehindert zu unterrichten. Die Pressefreiheit und die +Freiheit der Berichterstattung durch Rundfunk und Film werden +gewährleistet. Eine Zensur findet nicht statt. +(2) Diese Rechte finden ihre Schranken in den Vorschriften der +allgemeinen Gesetze, den gesetzlichen Bestimmungen zum Schutze der +Jugend und in dem Recht der persönlichen Ehre. +(3) Kunst und Wissenschaft, Forschung und Lehre sind frei. Die +Freiheit der Lehre entbindet nicht von der Treue zur Verfassung. +% +Grundgesetz Artikel 6 +(1) Ehe und Familie stehen unter dem besonderen Schutze der staatlichen Ordnung. +(2) Pflege und Erziehung der Kinder sind das natürliche Recht der +Eltern und die zuvörderst ihnen obliegende Pflicht. Über ihre +Betätigung wacht die staatliche Gemeinschaft. +(3) Gegen den Willen der Erziehungsberechtigten dürfen Kinder nur auf +Grund eines Gesetzes von der Familie getrennt werden, wenn die +Erziehungsberechtigten versagen oder wenn die Kinder aus anderen +Gründen zu verwahrlosen drohen. +(4) Jede Mutter hat Anspruch auf den Schutz und die Fürsorge der Gemeinschaft. +(5) Den unehelichen Kindern sind durch die Gesetzgebung die gleichen +Bedingungen für ihre leibliche und seelische Entwicklung und ihre +Stellung in der Gesellschaft zu schaffen wie den ehelichen Kindern. +% +Grundgesetz Artikel 7 +(1) Das gesamte Schulwesen steht unter der Aufsicht des Staates. +(2) Die Erziehungsberechtigten haben das Recht, über die Teilnahme des +Kindes am Religionsunterricht zu bestimmen. +(3) Der Religionsunterricht ist in den öffentlichen Schulen mit +Ausnahme der bekenntnisfreien Schulen ordentliches +Lehrfach. Unbeschadet des staatlichen Aufsichtsrechtes wird der +Religionsunterricht in Übereinstimmung mit den Grundsätzen der +Religionsgemeinschaften erteilt. Kein Lehrer darf gegen seinen Willen +verpflichtet werden, Religionsunterricht zu erteilen. +(4) Das Recht zur Errichtung von privaten Schulen wird +gewährleistet. Private Schulen als Ersatz für öffentliche Schulen +bedürfen der Genehmigung des Staates und unterstehen den +Landesgesetzen. Die Genehmigung ist zu erteilen, wenn die privaten +Schulen in ihren Lehrzielen und Einrichtungen sowie in der +wissenschaftlichen Ausbildung ihrer Lehrkräfte nicht hinter den +öffentlichen Schulen zurückstehen und eine Sonderung der Schüler nach +den Besitzverhältnissen der Eltern nicht gefördert wird. Die +Genehmigung ist zu versagen, wenn die wirtschaftliche und rechtliche +Stellung der Lehrkräfte nicht genügend gesichert ist. +(5) Eine private Volksschule ist nur zuzulassen, wenn die +Unterrichtsverwaltung ein besonderes pädagogisches Interesse anerkennt +oder, auf Antrag von Erziehungsberechtigten, wenn sie als +Gemeinschaftsschule, als Bekenntnis- oder Weltanschauungsschule +errichtet werden soll und eine öffentliche Volksschule dieser Art in +der Gemeinde nicht besteht. +(6) Vorschulen bleiben aufgehoben. +% +Grundgesetz Artikel 8 +(1) Alle Deutschen haben das Recht, sich ohne Anmeldung oder Erlaubnis +friedlich und ohne Waffen zu versammeln. +(2) Für Versammlungen unter freiem Himmel kann dieses Recht durch +Gesetz oder auf Grund eines Gesetzes beschränkt werden. +% +Grundgesetz Artikel 9 +(1) Alle Deutschen haben das Recht, Vereine und Gesellschaften zu bilden. +(2) Vereinigungen, deren Zwecke oder deren Tätigkeit den Strafgesetzen +zuwiderlaufen oder die sich gegen die verfassungsmäßige Ordnung oder +gegen den Gedanken der Völkerverständigung richten, sind verboten. +(3) Das Recht, zur Wahrung und Förderung der Arbeits- und +Wirtschaftsbedingungen Vereinigungen zu bilden, ist für jedermann und +für alle Berufe gewährleistet. Abreden, die dieses Recht einschränken +oder zu behindern suchen, sind nichtig, hierauf gerichtete Maßnahmen +sind rechtswidrig. Maßnahmen nach den Artikeln 12a, 35 Abs. 2 und 3, +Artikel 87a Abs. 4 und Artikel 91 dürfen sich nicht gegen +Arbeitskämpfe richten, die zur Wahrung und Förderung der Arbeits- und +Wirtschaftsbedingungen von Vereinigungen im Sinne des Satzes 1 geführt +werden. +% +Grundgesetz Artikel 10 +(1) Das Briefgeheimnis sowie das Post- und Fernmeldegeheimnis sind +unverletzlich. +(2) Beschränkungen dürfen nur auf Grund eines Gesetzes angeordnet +werden. Dient die Beschränkung dem Schutze der freiheitlichen +demokratischen Grundordnung oder des Bestandes oder der Sicherung des +Bundes oder eines Landes, so kann das Gesetz bestimmen, daß sie dem +Betroffenen nicht mitgeteilt wird und daß an die Stelle des +Rechtsweges die Nachprüfung durch von der Volksvertretung bestellte +Organe und Hilfsorgane tritt. +% +Grundgesetz Artikel 11 +(1) Alle Deutschen genießen Freizügigkeit im ganzen Bundesgebiet. +(2) Dieses Recht darf nur durch Gesetz oder auf Grund eines Gesetzes +und nur für die Fälle eingeschränkt werden, in denen eine ausreichende +Lebensgrundlage nicht vorhanden ist und der Allgemeinheit daraus +besondere Lasten entstehen würden oder in denen es zur Abwehr einer +drohenden Gefahr für den Bestand oder die freiheitliche demokratische +Grundordnung des Bundes oder eines Landes, zur Bekämpfung von +Seuchengefahr, Naturkatastrophen oder besonders schweren +Unglücksfällen, zum Schutze der Jugend vor Verwahrlosung oder um +strafbaren Handlungen vorzubeugen, erforderlich ist. +% +Grundgesetz Artikel 12 +(1) Alle Deutschen haben das Recht, Beruf, Arbeitsplatz und +Ausbildungsstätte frei zu wählen. Die Berufsausübung kann durch Gesetz +oder auf Grund eines Gesetzes geregelt werden. +(2) Niemand darf zu einer bestimmten Arbeit gezwungen werden, außer im +Rahmen einer herkömmlichen allgemeinen, für alle gleichen öffentlichen +Dienstleistungspflicht. +(3) Zwangsarbeit ist nur bei einer gerichtlich angeordneten +Freiheitsentziehung zulässig. +% +Grundgesetz Artikel 12a +(1) Männer können vom vollendeten achtzehnten Lebensjahr an zum Dienst +in den Streitkräften, im Bundesgrenzschutz oder in einem +Zivilschutzverband verpflichtet werden. +(2) Wer aus Gewissensgründen den Kriegsdienst mit der Waffe +verweigert, kann zu einem Ersatzdienst verpflichtet werden. Die Dauer +des Ersatzdienstes darf die Dauer des Wehrdienstes nicht +übersteigen. Das Nähere regelt ein Gesetz, das die Freiheit der +Gewissensentscheidung nicht beeinträchtigen darf und auch eine +Möglichkeit des Ersatzdienstes vorsehen muß, die in keinem +Zusammenhang mit den Verbänden der Streitkräfte und des +Bundesgrenzschutzes steht. +(3) Wehrpflichtige, die nicht zu einem Dienst nach Absatz 1 oder 2 +herangezogen sind, können im Verteidigungsfalle durch Gesetz oder auf +Grund eines Gesetzes zu zivilen Dienstleistungen für Zwecke der +Verteidigung einschließlich des Schutzes der Zivilbevölkerung in +Arbeitsverhältnisse verpflichtet werden; Verpflichtungen in +öffentlich-rechtliche Dienstverhältnisse sind nur zur Wahrnehmung +polizeilicher Aufgaben oder solcher hoheitlichen Aufgaben der +öffentlichen Verwaltung, die nur in einem öffentlich-rechtlichen +Dienstverhältnis erfüllt werden können, zulässig. Arbeitsverhältnisse +nach Satz 1 können bei den Streitkräften, im Bereich ihrer Versorgung +sowie bei der öffentlichen Verwaltung begründet werden; +Verpflichtungen in Arbeitsverhältnisse im Bereiche der Versorgung der +Zivilbevölkerung sind nur zulässig, um ihren lebensnotwendigen Bedarf +zu decken oder ihren Schutz sicherzustellen. +(4) Kann im Verteidigungsfalle der Bedarf an zivilen Dienstleistungen +im zivilen Sanitäts- und Heilwesen sowie in der ortsfesten +militärischen Lazarettorganisation nicht auf freiwilliger Grundlage +gedeckt werden, so können Frauen vom vollendeten achtzehnten bis zum +vollendeten fünfundfünfzigsten Lebensjahr durch Gesetz oder auf Grund +eines Gesetzes zu derartigen Dienstleistungen herangezogen werden. Sie +dürfen auf keinen Fall zum Dienst mit der Waffe verpflichtet werden. +(5) Für die Zeit vor dem Verteidigungsfalle können Verpflichtungen +nach Absatz 3 nur nach Maßgabe des Artikels 80a Abs. 1 begründet +werden. Zur Vorbereitung auf Dienstleistungen nach Absatz 3, für die +besondere Kenntnisse oder Fertigkeiten erforderlich sind, kann durch +Gesetz oder auf Grund eines Gesetzes die Teilnahme an +Ausbildungsveranstaltungen zur Pflicht gemacht werden. Satz 1 findet +insoweit keine Anwendung. +(6) Kann im Verteidigungsfalle der Bedarf an Arbeitskräften für die in +Absatz 3 Satz 2 genannten Bereiche auf freiwilliger Grundlage nicht +gedeckt werden, so kann zur Sicherung dieses Bedarfs die Freiheit der +Deutschen, die Ausübung eines Berufs oder den Arbeitsplatz aufzugeben, +durch Gesetz oder auf Grund eines Gesetzes eingeschränkt werden. Vor +Eintritt des Verteidigungsfalles gilt Absatz 5 Satz 1 entsprechend. +% +Grundgesetz Artikel 13 +(1) Die Wohnung ist unverletzlich. +(2) Durchsuchungen dürfen nur durch den Richter, bei Gefahr im Verzuge +auch durch die in den Gesetzen vorgesehenen anderen Organe angeordnet +und nur in der dort vorgeschriebenen Form durchgeführt werden. +(3) Begründen bestimmte Tatsachen den Verdacht, daß jemand eine durch +Gesetz einzeln bestimmte besonders schwere Straftat begangen hat, so +dürfen zur Verfolgung der Tat auf Grund richterlicher Anordnung +technische Mittel zur akustischen Überwachung von Wohnungen, in denen +der Beschuldigte sich vermutlich aufhält, eingesetzt werden, wenn die +Erforschung des Sachverhalts auf andere Weise unverhältnismäßig +erschwert oder aussichtslos wäre. Die Maßnahme ist zu befristen. Die +Anordnung erfolgt durch einen mit drei Richtern besetzten +Spruchkörper. Bei Gefahr im Verzuge kann sie auch durch einen +einzelnen Richter getroffen werden. +(4) Zur Abwehr dringender Gefahren für die öffentliche Sicherheit, +insbesondere einer gemeinen Gefahr oder einer Lebensgefahr, dürfen +technische Mittel zur Überwachung von Wohnungen nur auf Grund +richterlicher Anordnung eingesetzt werden. Bei Gefahr im Verzuge kann +die Maßnahme auch durch eine andere gesetzlich bestimmte Stelle +angeordnet werden; eine richterliche Entscheidung ist unverzüglich +nachzuholen. +(5) Sind technische Mittel ausschließlich zum Schutze der bei einem +Einsatz in Wohnungen tätigen Personen vorgesehen, kann die Maßnahme +durch eine gesetzlich bestimmte Stelle angeordnet werden. Eine +anderweitige Verwertung der hierbei erlangten Erkenntnisse ist nur zum +Zwecke der Strafverfolgung oder der Gefahrenabwehr und nur zulässig, +wenn zuvor die Rechtmäßigkeit der Maßnahme richterlich festgestellt +ist; bei Gefahr im Verzuge ist die richterliche Entscheidung +unverzüglich nachzuholen. +(6) Die Bundesregierung unterrichtet den Bundestag jährlich über den +nach Absatz 3 sowie über den im Zuständigkeitsbereich des Bundes nach +Absatz 4 und, soweit richterlich überprüfungsbedürftig, nach Absatz 5 +erfolgten Einsatz technischer Mittel. Ein vom Bundestag gewähltes +Gremium übt auf der Grundlage dieses Berichts die parlamentarische +Kontrolle aus. Die Länder gewährleisten eine gleichwertige +parlamentarische Kontrolle. +(7) Eingriffe und Beschränkungen dürfen im übrigen nur zur Abwehr +einer gemeinen Gefahr oder einer Lebensgefahr für einzelne Personen, +auf Grund eines Gesetzes auch zur Verhütung dringender Gefahren für +die öffentliche Sicherheit und Ordnung, insbesondere zur Behebung der +Raumnot, zur Bekämpfung von Seuchengefahr oder zum Schutze gefährdeter +Jugendlicher vorgenommen werden. +% +Grundgesetz Artikel 14 +(1) Das Eigentum und das Erbrecht werden gewährleistet. Inhalt und +Schranken werden durch die Gesetze bestimmt. +(2) Eigentum verpflichtet. Sein Gebrauch soll zugleich dem Wohle der +Allgemeinheit dienen. +(3) Eine Enteignung ist nur zum Wohle der Allgemeinheit zulässig. Sie +darf nur durch Gesetz oder auf Grund eines Gesetzes erfolgen, das Art +und Ausmaß der Entschädigung regelt. Die Entschädigung ist unter +gerechter Abwägung der Interessen der Allgemeinheit und der +Beteiligten zu bestimmen. Wegen der Höhe der Entschädigung steht im +Streitfalle der Rechtsweg vor den ordentlichen Gerichten offen. +% +Grundgesetz Artikel 15 +Grund und Boden, Naturschätze und Produktionsmittel können zum Zwecke +der Vergesellschaftung durch ein Gesetz, das Art und Ausmaß der +Entschädigung regelt, in Gemeineigentum oder in andere Formen der +Gemeinwirtschaft überführt werden. Für die Entschädigung gilt Artikel +14 Abs. 3 Satz 3 und 4 entsprechend. +% +Grundgesetz Artikel 16 +(1) Die deutsche Staatsangehörigkeit darf nicht entzogen werden. Der +Verlust der Staatsangehörigkeit darf nur auf Grund eines Gesetzes und +gegen den Willen des Betroffenen nur dann eintreten, wenn der +Betroffene dadurch nicht staatenlos wird. +(2) Kein Deutscher darf an das Ausland ausgeliefert werden. Durch +Gesetz kann eine abweichende Regelung für Auslieferungen an einen +Mitgliedstaat der Europäischen Union oder an einen internationalen +Gerichtshof getroffen werden, soweit rechtsstaatliche Grundsätze +gewahrt sind. +% +Grundgesetz Artikel 16a +(1) Politisch Verfolgte genießen Asylrecht. +(2) Auf Absatz 1 kann sich nicht berufen, wer aus einem Mitgliedstaat +der Europäischen Gemeinschaften oder aus einem anderen Drittstaat +einreist, in dem die Anwendung des Abkommens über die Rechtsstellung +der Flüchtlinge und der Konvention zum Schutze der Menschenrechte und +Grundfreiheiten sichergestellt ist. Die Staaten außerhalb der +Europäischen Gemeinschaften, auf die die Voraussetzungen des Satzes 1 +zutreffen, werden durch Gesetz, das der Zustimmung des Bundesrates +bedarf, bestimmt. In den Fällen des Satzes 1 können +aufenthaltsbeendende Maßnahmen unabhängig von einem hiergegen +eingelegten Rechtsbehelf vollzogen werden. +(3) Durch Gesetz, das der Zustimmung des Bundesrates bedarf, können +Staaten bestimmt werden, bei denen auf Grund der Rechtslage, der +Rechtsanwendung und der allgemeinen politischen Verhältnisse +gewährleistet erscheint, daß dort weder politische Verfolgung noch +unmenschliche oder erniedrigende Bestrafung oder Behandlung +stattfindet. Es wird vermutet, daß ein Ausländer aus einem solchen +Staat nicht verfolgt wird, solange er nicht Tatsachen vorträgt, die +die Annahme begründen, daß er entgegen dieser Vermutung politisch +verfolgt wird. +(4) Die Vollziehung aufenthaltsbeendender Maßnahmen wird in den Fällen +des Absatzes 3 und in anderen Fällen, die offensichtlich unbegründet +sind oder als offensichtlich unbegründet gelten, durch das Gericht nur +ausgesetzt, wenn ernstliche Zweifel an der Rechtmäßigkeit der Maßnahme +bestehen; der Prüfungsumfang kann eingeschränkt werden und verspätetes +Vorbringen unberücksichtigt bleiben. Das Nähere ist durch Gesetz zu +bestimmen. +(5) Die Absätze 1 bis 4 stehen völkerrechtlichen Verträgen von +Mitgliedstaaten der Europäischen Gemeinschaften untereinander und mit +dritten Staaten nicht entgegen, die unter Beachtung der +Verpflichtungen aus dem Abkommen über die Rechtsstellung der +Flüchtlinge und der Konvention zum Schutze der Menschenrechte und +Grundfreiheiten, deren Anwendung in den Vertragsstaaten sichergestellt +sein muß, Zuständigkeitsregelungen für die Prüfung von Asylbegehren +einschließlich der gegenseitigen Anerkennung von Asylentscheidungen +treffen. +% +Grundgesetz Artikel 17 +Jedermann hat das Recht, sich einzeln oder in Gemeinschaft mit anderen +schriftlich mit Bitten oder Beschwerden an die zuständigen Stellen und +an die Volksvertretung zu wenden. +% +Grundgesetz Artikel 17a +(1) Gesetze über Wehrdienst und Ersatzdienst können bestimmen, daß für +die Angehörigen der Streitkräfte und des Ersatzdienstes während der +Zeit des Wehr- oder Ersatzdienstes das Grundrecht, seine Meinung in +Wort, Schrift und Bild frei zu äußern und zu verbreiten (Artikel 5 +Abs. 1 Satz 1 erster Halbsatz), das Grundrecht der +Versammlungsfreiheit (Artikel 8) und das Petitionsrecht (Artikel 17), +soweit es das Recht gewährt, Bitten oder Beschwerden in Gemeinschaft +mit anderen vorzubringen, eingeschränkt werden. +(2) Gesetze, die der Verteidigung einschließlich des Schutzes der +Zivilbevölkerung dienen, können bestimmen, daß die Grundrechte der +Freizügigkeit (Artikel 11) und der Unverletzlichkeit der Wohnung +(Artikel 13) eingeschränkt werden. +% +Grundgesetz Artikel 18 +Wer die Freiheit der Meinungsäußerung, insbesondere die Pressefreiheit +(Artikel 5 Abs. 1), die Lehrfreiheit (Artikel 5 Abs. 3), die +Versammlungsfreiheit (Artikel 8), die Vereinigungsfreiheit (Artikel +9), das Brief-, Post- und Fernmeldegeheimnis (Artikel 10), das +Eigentum (Artikel 14) oder das Asylrecht (Artikel 16a) zum Kampfe +gegen die freiheitliche demokratische Grundordnung mißbraucht, +verwirkt diese Grundrechte. Die Verwirkung und ihr Ausmaß werden durch +das Bundesverfassungsgericht ausgesprochen. +% +Grundgesetz Artikel 19 +(1) Soweit nach diesem Grundgesetz ein Grundrecht durch Gesetz oder +auf Grund eines Gesetzes eingeschränkt werden kann, muß das Gesetz +allgemein und nicht nur für den Einzelfall gelten. Außerdem muß das +Gesetz das Grundrecht unter Angabe des Artikels nennen. +(2) In keinem Falle darf ein Grundrecht in seinem Wesensgehalt +angetastet werden. +(3) Die Grundrechte gelten auch für inländische juristische Personen, +soweit sie ihrem Wesen nach auf diese anwendbar sind. +(4) Wird jemand durch die öffentliche Gewalt in seinen Rechten +verletzt, so steht ihm der Rechtsweg offen. Soweit eine andere +Zuständigkeit nicht begründet ist, ist der ordentliche Rechtsweg +gegeben. Artikel 10 Abs. 2 Satz 2 bleibt unberührt. +% -- 2.7.0 From 43e905fa16f08ca32bc5c7fb7c64f4fa702fc4c4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 13:16:50 +0200 Subject: [PATCH 181/213] share/skel/dot.profile: Change fortune file to "grundrechte" --- share/skel/dot.profile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/skel/dot.profile b/share/skel/dot.profile index ad66198..eb53895 100644 --- a/share/skel/dot.profile +++ b/share/skel/dot.profile @@ -21,4 +21,4 @@ PAGER=more; export PAGER # set ENV to a file invoked each time sh is started for interactive use. ENV=$HOME/.shrc; export ENV -if [ -x /usr/bin/fortune ] ; then /usr/bin/fortune freebsd-tips ; fi +if [ -x /usr/bin/fortune ] ; then /usr/bin/fortune grundrechte ; fi -- 2.7.0 From b3c0507a0c772491815bcb675b41a887fbfd0bd9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 18 Aug 2015 09:57:30 +0200 Subject: [PATCH 182/213] XEN: Note that ElectroBSD does not protect against compromised hosts --- sys/x86/xen/hvm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c index a8b14fd..07616c1 100644 --- a/sys/x86/xen/hvm.c +++ b/sys/x86/xen/hvm.c @@ -313,6 +313,10 @@ xen_hvm_init(enum xen_hvm_init_type init_type) panic("Unsupported HVM initialization type"); } + printf("ElectroBSD security advise: It looks like you may be clown computing.\n" + "Please note that the people controlling the 'cloud' this system is\n" + "running on can monitor and sabotage everything it is doing.\n"); + xen_vector_callback_enabled = 0; xen_hvm_set_callback(NULL); -- 2.7.0 From 8d0fde4e3a38743babf7384babb30c62b466c5fd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 19:53:09 +0200 Subject: [PATCH 183/213] top: Show ZFS ARC target size --- usr.bin/top/machine.c | 6 ++++-- usr.bin/top/top.local.1 | 12 +++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/usr.bin/top/machine.c b/usr.bin/top/machine.c index 20be935..8739bb7 100644 --- a/usr.bin/top/machine.c +++ b/usr.bin/top/machine.c @@ -180,9 +180,9 @@ char *memorynames[] = { "K Free", NULL }; -int arc_stats[7]; +int arc_stats[8]; char *arcnames[] = { - "K Total, ", "K MFU, ", "K MRU, ", "K Anon, ", "K Header, ", "K Other", + "K Total, ", "K MFU, ", "K MRU, ", "K Anon, ", "K Header, ", "K Other, ", "K Target", NULL }; @@ -544,6 +544,8 @@ get_system_info(struct system_info *si) arc_stats[4] = arc_stat + arc_stat2 >> 10; GETSYSCTL("kstat.zfs.misc.arcstats.other_size", arc_stat); arc_stats[5] = arc_stat >> 10; + GETSYSCTL("kstat.zfs.misc.arcstats.c", arc_stat); + arc_stats[6] = arc_stat >> 10; si->arc = arc_stats; } diff --git a/usr.bin/top/top.local.1 b/usr.bin/top/top.local.1 index 864ab8d..951754c 100644 --- a/usr.bin/top/top.local.1 +++ b/usr.bin/top/top.local.1 @@ -2,9 +2,9 @@ .SH "FreeBSD NOTES" .SH DESCRIPTION OF MEMORY -Mem: 9220K Active, 1M Inact, 3284K Wired, 1M Cache, 2M Buf, 1320K Free -ARC: 2048K Total, 342K MRU, 760K MFU, 272K Anon, 232K Header, 442K Other -Swap: 91M Total, 79M Free, 13% Inuse, 80K In, 104K Out +Mem: 387M Active, 499M Inact, 945M Wired, 8K Cache, 5528K Buf, 83M Free +ARC: 349M Total, 174M MFU, 94M MRU, 2192K Anon, 6488K Header, 73M Other, 350M Target +Swap: 2048M Total, 124M Used, 1924M Free, 6% Inuse .TP .B K: Kilobyte @@ -57,6 +57,12 @@ number of ARC bytes holding headers .TP .B Other miscellaneous ARC bytes +.TP +.B Target +ARC target size, that is the total amount of memory +the ARC considers usable for itself. If it's not equal +to the total size, the ARC will shrink or grow to reach +the target. .SS Swap Stats .TP .B Total: -- 2.7.0 From 77fd2e57812347218d2aa756c28656301ed5ff65 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 12:34:53 +0200 Subject: [PATCH 184/213] sys/vm: Stop increasing domain->vmd_pass after reaching the "maximum" value ... explicitly understood by vm_pageout_scan(). Should prevent (purely cosmetic) issues like: fk@r500 ~ $sudo /usr/src/share/dtrace/monitor-page-scanner [...] 2015 Sep 22 12:15:54: Scan goal 59: Invalid 2015 Sep 22 12:15:54: Scan goal 60: Invalid 2015 Sep 22 12:15:55: Scan goal 61: Invalid 2015 Sep 22 12:15:55: Scan goal 62: Invalid 2015 Sep 22 12:15:55: Scan goals in the previous minute: Launder dirty pages 1 Pageout dirty pages 1 Move inactive to cache or free 2 Invalid 5 Update active LRU/deactivate pages 28 2015 Sep 22 12:15:55: Seconds since last 'Move inactive to cache or free' pass: 30 2015 Sep 22 12:15:55: Seconds since last 'Launder dirty pages' pass: 30 2015 Sep 22 12:15:55: Seconds since last 'Pageout dirty pages' pass: 30 --- sys/vm/vm_pageout.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index d38e985..5d6d512 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -867,6 +867,7 @@ unlock_mp: return (error); } +#define VMD_PASS_MAX 3 /* * vm_pageout_scan does the dirty work for the pageout daemon. * @@ -886,6 +887,9 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) int vnodes_skipped; boolean_t pageout_ok, queues_locked; + KASSERT(pass <= VMD_PASS_MAX, + ("vm_pageout_scan: Invalid pass code %d", pass)); + /* * If we need to reclaim memory ask kernel caches to return * some. We rate limit to avoid thrashing. @@ -1588,7 +1592,8 @@ vm_pageout_worker(void *arg) } if (vm_pages_needed) { vm_cnt.v_pdwakeups++; - domain->vmd_pass++; + if (domain->vmd_pass < VMD_PASS_MAX) + domain->vmd_pass++; } else domain->vmd_pass = 0; mtx_unlock(&vm_page_queue_free_mtx); -- 2.7.0 From 9636b6becc7e2ecb3864aaa38f4989cea5b4629f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 16:36:46 +0200 Subject: [PATCH 185/213] vm_pageout_scan(): Add SDT probes to make a couple of internal variables visible --- sys/vm/vm_pageout.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 5d6d512..afb5557 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -141,6 +141,15 @@ SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, SDT_PROVIDER_DEFINE(vm); SDT_PROBE_DEFINE(vm, , , vm__lowmem_cache); SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan); +SDT_PROBE_DEFINE4(vm, , , before__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int deficit"); +SDT_PROBE_DEFINE5(vm, , , after__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int addl_page_shortage", + "int vnodes_skipped"); +SDT_PROBE_DEFINE3(vm, , , before__active__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage"); +SDT_PROBE_DEFINE3(vm, , , after__active__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage"); #if !defined(NO_SWAPPING) /* the kernel process "vm_daemon"*/ @@ -945,6 +954,9 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) vnodes_skipped = 0; + SDT_PROBE4(vm, , , before__inactive__scan, vmd, pass, page_shortage, + deficit); + /* * Start scanning the inactive queue for pages we can move to the * cache or free. The scan will stop when the target is reached or @@ -1172,6 +1184,9 @@ relock_queues: } vm_pagequeue_unlock(pq); + SDT_PROBE5(vm, , , after__inactive__scan, vmd, pass, page_shortage, + addl_page_shortage, vnodes_skipped); + #if !defined(NO_SWAPPING) /* * Wakeup the swapout daemon if we didn't cache or free the targeted @@ -1220,6 +1235,8 @@ relock_queues: if (min_scan > 0 || (page_shortage > 0 && maxscan > 0)) vmd->vmd_last_active_scan = scan_tick; + SDT_PROBE3(vm, , , before__active__scan, vmd, pass, page_shortage); + /* * Scan the active queue for pages that can be deactivated. Update * the per-page activity counter and use it to identify deactivation @@ -1294,6 +1311,9 @@ relock_queues: vm_page_unlock(m); } vm_pagequeue_unlock(pq); + + SDT_PROBE3(vm, , , after__active__scan, vmd, pass, page_shortage); + #if !defined(NO_SWAPPING) /* * Idle process swapout -- run once per second. -- 2.7.0 From 738c743f52891c52bdd376c770756a287c0e58de Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 16:05:49 +0200 Subject: [PATCH 186/213] sys/vm: Limit the inactive memory more aggressively Currently the ZFS ARC does not take the inactive memory into account when calculating its target size. If there's no limit for the inactive pages, the ARC may shrink to its own limit while the number of inactive pages continues to grow: last pid: 28429; load averages: 0.48, 0.46, 0.41 up 0+03:39:07 17:24:59 91 processes: 2 running, 88 sleeping, 1 waiting CPU: 1.4% user, 0.0% nice, 12.7% system, 0.2% interrupt, 85.7% idle Mem: 396M Active, 489M Inact, 986M Wired, 292K Cache, 5202K Buf, 43M Free ARC: 351M Total, 90M MFU, 44M MRU, 6839K Anon, 7810K Header, 203M Other, 350M Target Swap: 2048M Total, 99M Used, 1949M Free, 4% Inuse PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND 11 root 2 155 ki31 0K 32K RUN 0 377:37 170.34% idle 26625 fk 17 36 0 175M 24504K uwait 1 0:09 8.40% git 0 root 468 -16 0 0K 7488K swapin 1 3:29 6.26% kernel 22 root 1 20 - 0K 16K geli:w 1 4:16 5.06% g_eli[1] ada0s1d [...] 2015 Sep 21 17:24:58: Scan goals in the previous minute: Update active LRU/deactivate pages 60 2015 Sep 21 17:24:58: Seconds since last 'Move inactive to cache or free' pass: 1477 2015 Sep 21 17:24:58: Seconds since last 'Launder dirty pages' pass: 9273 With this commit, the system can be configured to let the ARC indirectly put pressure on the inactive memory until a given target is reached. A couple of sysctls can and should be used to tune the limits, as the defaults currently aren't auto-tuned. Note that suboptimal tuning can result in excessive paging. Example /etc/sysctl.conf excerpt that appears to work reasonably well for an ElectroBSD development system with 2 GB of RAM: # Set the free page target for the ZFS ARC slightly below # the autotuned vm.v_free_target (10479) so the ARC shrinks # before the system starts paging. vfs.zfs.arc_free_target=10000 # If we come too close to the vm.v_free_target, start freeing # inactive pages above the ceiling of the inactive page # target plus the offset_f. vm.inactive_page_limit_offset_i=40000 vm.inactive_page_limit_offset_f=10000 # Free up to 10000 inactive pages in a row, before checking # if we should continue to do so. vm.inactive_pages_to_free_max=10000 Screenshot: https://www.fabiankeil.de/bilder/electrobsd/kernel-compilation-with-inactive-page-limit-enabled.png --- sys/vm/vm_pageout.c | 125 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 109 insertions(+), 16 deletions(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index afb5557..019e1f0 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -141,15 +141,17 @@ SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, SDT_PROVIDER_DEFINE(vm); SDT_PROBE_DEFINE(vm, , , vm__lowmem_cache); SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan); -SDT_PROBE_DEFINE4(vm, , , before__inactive__scan, "struct vm_domain *vmd", - "int pass", "int page_shortage", "int deficit"); -SDT_PROBE_DEFINE5(vm, , , after__inactive__scan, "struct vm_domain *vmd", +SDT_PROBE_DEFINE5(vm, , , before__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int deficit", "int inactive_page_surplus"); +SDT_PROBE_DEFINE6(vm, , , after__inactive__scan, "struct vm_domain *vmd", "int pass", "int page_shortage", "int addl_page_shortage", - "int vnodes_skipped"); + "int vnodes_skipped", "int inactive_page_surplus"); SDT_PROBE_DEFINE3(vm, , , before__active__scan, "struct vm_domain *vmd", "int pass", "int page_shortage"); SDT_PROBE_DEFINE3(vm, , , after__active__scan, "struct vm_domain *vmd", "int pass", "int page_shortage"); +SDT_PROBE_DEFINE3(vm, , , checked__inactive__pages, "int pages_to_free", + "int pages_above_limit", "int enforced_limit"); #if !defined(NO_SWAPPING) /* the kernel process "vm_daemon"*/ @@ -230,6 +232,30 @@ SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts, SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts, CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages"); +static int inactive_page_limit_enabled = 1; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_enabled, CTLFLAG_RW, + &inactive_page_limit_enabled, 0, + "Free inactive pages above the target more aggressively. " + "Values: 0 (disabled), 1 (soft mode, only apply limit if free " + "page count is low), 2 (ignore free count)"); + +static int inactive_page_limit_offset_i = 0; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_offset_i, CTLFLAG_RW, + &inactive_page_limit_offset_i, 0, + "Number of inactive pages relative to the inactive target " + "required for inactive pages to be freed."); + +static int inactive_page_limit_offset_f = 0; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_offset_f, CTLFLAG_RW, + &inactive_page_limit_offset_f, 0, + "Number of free pages relative to the free target required for " + "the inactive memory limit to be applied."); + +static int inactive_pages_to_free_max = 1000; +SYSCTL_INT(_vm, OID_AUTO, inactive_pages_to_free_max, CTLFLAG_RW, + &inactive_pages_to_free_max, 0, + "Maximum number of inactive pages above the target to free at once."); + static int pageout_lock_miss; SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss, CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout"); @@ -876,6 +902,41 @@ unlock_mp: return (error); } +static int +vm_pageout_get_inactive_page_surplus(void) +{ + int pages_to_free; + int pages_above_limit; + int enforced_limit; + + /* Return early so the DTrace probe does not fire. */ + if (!inactive_page_limit_enabled) + return (0); + + enforced_limit = vm_cnt.v_inactive_target + inactive_page_limit_offset_i; + pages_above_limit = vm_cnt.v_inactive_count - enforced_limit; + + /* + * We want to free inactive pages if there actually are + * inactive pages above the limit and we are either using + * a hard limit, or the number of free pages is below the + * free page limit. + */ + if ((pages_above_limit > 0) && + ((inactive_page_limit_enabled == 2) || + (vm_paging_target() + inactive_page_limit_offset_f > 0))) { + pages_to_free = imin(inactive_pages_to_free_max, + pages_above_limit); + } else { + pages_to_free = 0; + } + + SDT_PROBE3(vm, , , checked__inactive__pages, pages_to_free, + pages_above_limit, enforced_limit); + + return (pages_to_free); +} + #define VMD_PASS_MAX 3 /* * vm_pageout_scan does the dirty work for the pageout daemon. @@ -894,6 +955,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) int act_delta, addl_page_shortage, deficit, error, maxlaunder, maxscan; int page_shortage, scan_tick, scanned, starting_page_shortage; int vnodes_skipped; + int inactive_page_surplus; boolean_t pageout_ok, queues_locked; KASSERT(pass <= VMD_PASS_MAX, @@ -933,8 +995,9 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) if (pass > 0) { deficit = atomic_readandclear_int(&vm_pageout_deficit); page_shortage = vm_paging_target() + deficit; + inactive_page_surplus = vm_pageout_get_inactive_page_surplus(); } else - page_shortage = deficit = 0; + page_shortage = deficit = inactive_page_surplus = 0; starting_page_shortage = page_shortage; /* @@ -952,10 +1015,18 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) if (pass > 1) maxlaunder = 10000; + /* + * Prevent laundering if there's no page shortage and we are + * merely trying to free inactive pages. Otherwise we may end + * up swapping before it's really necessary. + */ + if (page_shortage <= 0) + maxlaunder = 0; + vnodes_skipped = 0; - SDT_PROBE4(vm, , , before__inactive__scan, vmd, pass, page_shortage, - deficit); + SDT_PROBE5(vm, , , before__inactive__scan, vmd, pass, page_shortage, + deficit, inactive_page_surplus); /* * Start scanning the inactive queue for pages we can move to the @@ -969,7 +1040,8 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) vm_pagequeue_lock(pq); queues_locked = TRUE; for (m = TAILQ_FIRST(&pq->pq_pl); - m != NULL && maxscan-- > 0 && page_shortage > 0; + m != NULL && maxscan-- > 0 && + (page_shortage > 0 || inactive_page_surplus > 0); m = next) { vm_pagequeue_assert_locked(pq); KASSERT(queues_locked, ("unlocked queues")); @@ -1106,6 +1178,7 @@ free_page: vm_page_free(m); PCPU_INC(cnt.v_dfree); --page_shortage; + --inactive_page_surplus; } else if ((object->flags & OBJ_DEAD) != 0) { /* * Leave dirty pages from dead objects at the front of @@ -1133,13 +1206,19 @@ requeue_page: vm_pagequeue_lock(pq); queues_locked = TRUE; vm_page_requeue_locked(m); - } else if (maxlaunder > 0) { + } else if (maxlaunder > 0 && page_shortage > 0) { /* - * We always want to try to flush some dirty pages if - * we encounter them, to keep the system stable. + * As long as there is a page shortage, we try to + * flush some dirty pages if we encounter them, to + * keep the system stable. * Normally this number is small, but under extreme * pressure where there are insufficient clean pages * on the inactive queue, we may have to go all out. + * + * XXX: We probably should not bother laundering + * until we know that there might be a chance + * that we will not be able to free the required + * amount of pages to take care of the page shortage. */ if (object->type != OBJT_SWAP && @@ -1184,15 +1263,27 @@ relock_queues: } vm_pagequeue_unlock(pq); - SDT_PROBE5(vm, , , after__inactive__scan, vmd, pass, page_shortage, - addl_page_shortage, vnodes_skipped); + /* + * If the page shortage has been taken care of, or if we were + * just trying to free surplus inactive pages, the locked pages + * are more or less meaningless. Reset the counter to prevent + * pointless swapping. + */ + if (page_shortage <= 0) + addl_page_shortage = 0; + + SDT_PROBE6(vm, , , after__inactive__scan, vmd, pass, page_shortage, + addl_page_shortage, vnodes_skipped, inactive_page_surplus); #if !defined(NO_SWAPPING) /* - * Wakeup the swapout daemon if we didn't cache or free the targeted - * number of pages. + * Wakeup the swapout daemon if we didn't cache or free the + * targeted number of pages and we are either desperate or there + * are no inactive pages to free left (in which case we will be + * desperate soon enough). */ - if (vm_swap_enabled && page_shortage > 0) + if (vm_swap_enabled && page_shortage > 0 && + (pass > 1 || !vm_pageout_get_inactive_page_surplus())) vm_req_vmdaemon(VM_SWAP_NORMAL); #endif @@ -1586,6 +1677,8 @@ vm_pageout_worker(void *arg) vm_pages_needed = 0; wakeup(&vm_cnt.v_free_count); } + if (vm_pageout_get_inactive_page_surplus() > 0) + vm_pages_needed = 1; if (vm_pages_needed) { /* * We're still not done. Either vm_pages_needed was -- 2.7.0 From 3fd326ac26d3a9eac60b9cf2ee5ddc19c0b81504 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 19:10:51 +0200 Subject: [PATCH 187/213] share/dtrace: Add monitor-page-scanner ... which, who would have guessed it, monitors the vm page scanner. It's useful to tune the sysctls for the inactive page limit. --- share/dtrace/monitor-page-scanner | 168 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 share/dtrace/monitor-page-scanner diff --git a/share/dtrace/monitor-page-scanner b/share/dtrace/monitor-page-scanner new file mode 100755 index 0000000..85537ca --- /dev/null +++ b/share/dtrace/monitor-page-scanner @@ -0,0 +1,168 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * monitor-page-scanner + * + * Traces the vm page scanner. + * + * Relies on SDT probes that currrently are not part of vanilla FreeBSD. + * + * Copyright (c) 2015 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + goal[0] = "Update active LRU/deactivate pages"; + goal[1] = "Move inactive to cache or free"; + goal[2] = "Launder dirty pages"; + goal[3] = "Pageout dirty pages"; + start_time = walltimestamp; + + min_pass = 2; + + printf("%Y: Monitoring the page scanner. Minimum pass value to show 'boring' scans ", walltimestamp); + printf("without memory pressure or inactive page surplus: %d (%s). Press CTRL-C to abort.\n", + min_pass, goal[min_pass]); +} + +vm:kernel::checked-inactive-pages +{ + this->to_free = (int)arg0; +} + +vm:kernel::checked-inactive-pages +/this->to_free/ +{ + this->above_limit = (int)arg1; + + printf("%Y: %s: Inactive page surplus: %d, above limit: %d.\n", + walltimestamp, probename, this->to_free, this->above_limit); +} + +vm:kernel::before-inactive-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; + this->inactive_page_surplus = (int)arg4; + + /* + * Every pass code above 3 is treated like 3, + * adjust index accordingly. + */ + this->goal_index = (this->pass <= 3) ? this->pass : 3; + this->goal = goal[this->goal_index]; + @goals[this->goal] = count(); + @goals_total[this->goal] = count(); + last_pass[this->pass] = timestamp; +} + +vm:kernel::before-inactive-scan +/(this->pass >= min_pass) || (this->page_shortage > 0) +|| (this->inactive_page_surplus > 0)/ +{ + /* this->vmd = (struct vm_domain *)arg0; */ + this->deficit = (int)arg3; + + printf("%Y: %s: Scan goal %d: %s.\n", + walltimestamp, probename, this->pass, this->goal); + printf("%Y: %s: Page shortage: %d, inactive page surplus: %d\n", + walltimestamp, probename, this->page_shortage, this->inactive_page_surplus); + printf("%Y: v_free_target: %d, v_free_count: %d, v_cache_count: %d.\n", + walltimestamp, `vm_cnt.v_free_target, `vm_cnt.v_free_count, `vm_cnt.v_cache_count); + printf("%Y: v_inactive_target: %d. v_inactive_count: %d. deficit: %d\n", + walltimestamp, `vm_cnt.v_inactive_target, `vm_cnt.v_inactive_count, this->deficit); +} + +vm:kernel::after-inactive-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; + this->addl_page_shortage = (int)arg3; + this->vnodes_skipped = (int)arg4; + this->inactive_page_surplus = (int)arg5; +} + +vm:kernel::after-inactive-scan +/(this->pass >= min_pass) || (this->page_shortage > 0) || +(this->addl_page_shortage > 0) || (this->vnodes_skipped > 0) || +(this->inactive_page_surplus > 0)/ +{ + printf("%Y: %s pass %d: page shortage: %d, inactive page surplus: %d, addl shortage: %d, vnodes skipped: %d.\n", + walltimestamp, probename, this->pass, this->page_shortage, + this->inactive_page_surplus, this->addl_page_shortage, this->vnodes_skipped); +} + +vm:kernel::before-active-scan, +vm:kernel::after-active-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; +} + +vm:kernel::before-active-scan, +vm:kernel::after-active-scan +/(this->pass >= min_pass) || (this->page_shortage > 0)/ +{ + printf("%Y: %s pass %d: page_shortage: %d.\n", + walltimestamp, probename, this->pass, this->page_shortage); +} + +tick-60s +{ + printf("%Y: Scan goals in the previous minute:", walltimestamp); + printa(@goals); + trunc(@goals); +} + +tick-60s +/last_pass[1]/ +{ + this->pass = 1; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +tick-60s +/last_pass[2]/ +{ + this->pass = 2; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +tick-60s +/last_pass[3]/ +{ + this->pass = 3; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +END +{ + printf("%Y: Scan goals since start of script at %Y:", + walltimestamp, start_time); + printa(@goals_total); + + /* Clear aggregates so DTrace does not show them again. */ + trunc(@goals_total); + trunc(@goals); +} -- 2.7.0 From ab6407da73c6bc99956004d46e4a900cbb55b7be Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 13 Oct 2015 18:19:42 +0200 Subject: [PATCH 188/213] libdtrace: Replace another timestamp related assert() with a warning It triggered a lot less often, but was still annoying. Example: 2015 Oct 13 18:01:06: v_free_target: 20888, v_free_count: 44001, v_cache_count: 0. 2015 Oct 13 18:01:06: v_inactive_target: 31332. v_inactive_count: 364311. deficit: 0 2015 Oct 13 18:01:06: after-inactive-scan pass 3: page shortage: -23113, inactive page surplus: 10000, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:06: before-active-scan pass 3: page_shortage: -356092. 2015 Oct 13 18:01:06: after-active-scan pass 3: page_shortage: -356092. 2015 Oct 13 18:01:06: checked-inactive-pages: Inactive page surplus: 10000, above limit: 252979. dtrace: cpu clocks out of sync (18128465664478 < 18128471798881; offset: 6134403). Results may be incorrect! dtrace: cpu clocks out of sync (18129463671844 < 18129469778266; offset: 6106422). Results may be incorrect! 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253446. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253446. 2015 Oct 13 18:01:07: before-inactive-scan: Scan goal 1: Move inactive to cache or free. 2015 Oct 13 18:01:07: before-inactive-scan: Page shortage: -37096, inactive page surplus: 10000 2015 Oct 13 18:01:07: v_free_target: 20888, v_free_count: 57984, v_cache_count: 0. 2015 Oct 13 18:01:07: v_inactive_target: 31332. v_inactive_count: 364778. deficit: 0 2015 Oct 13 18:01:07: after-inactive-scan pass 1: page shortage: -37203, inactive page surplus: 9893, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253250. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253250. 2015 Oct 13 18:01:07: before-inactive-scan: Scan goal 2: Launder dirty pages. 2015 Oct 13 18:01:07: before-inactive-scan: Page shortage: -37191, inactive page surplus: 10000 2015 Oct 13 18:01:07: v_free_target: 20888, v_free_count: 58079, v_cache_count: 0. 2015 Oct 13 18:01:07: v_inactive_target: 31332. v_inactive_count: 364582. deficit: 0 2015 Oct 13 18:01:07: after-inactive-scan pass 2: page shortage: -37194, inactive page surplus: 9997, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:07: before-active-scan pass 2: page_shortage: -370441. 2015 Oct 13 18:01:07: after-active-scan pass 2: page_shortage: -370441. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253268. dtrace: cpu clocks out of sync (18130462652022 < 18130468761237; offset: 6109215). Results may be incorrect! 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253331. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253331. 2015 Oct 13 18:01:08: before-inactive-scan: Scan goal 1: Move inactive to cache or free. 2015 Oct 13 18:01:08: before-inactive-scan: Page shortage: -34625, inactive page surplus: 10000 2015 Oct 13 18:01:08: v_free_target: 20888, v_free_count: 55513, v_cache_count: 0. 2015 Oct 13 18:01:08: v_inactive_target: 31332. v_inactive_count: 364663. deficit: 0 2015 Oct 13 18:01:08: after-inactive-scan pass 1: page shortage: -34645, inactive page surplus: 9980, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253286. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253286. 2015 Oct 13 18:01:08: before-inactive-scan: Scan goal 2: Launder dirty pages. 2015 Oct 13 18:01:08: before-inactive-scan: Page shortage: -34395, inactive page surplus: 10000 2015 Oct 13 18:01:08: v_free_target: 20888, v_free_count: 55283, v_cache_count: 0. 2015 Oct 13 18:01:08: v_inactive_target: 31332. v_inactive_count: 364618. deficit: 0 2015 Oct 13 18:01:08: after-inactive-scan pass 2: page shortage: -34396, inactive page surplus: 9999, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:08: before-active-scan pass 2: page_shortage: -367391. 2015 Oct 13 18:01:08: after-active-scan pass 2: page_shortage: -367391. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253285. 2015 Oct 13 18:01:09: checked-inactive-pages: Inactive page surplus: 10000, above limit: 256158. 2015 Oct 13 18:01:09: before-inactive-scan: Scan goal 3: Pageout dirty pages. 2015 Oct 13 18:01:09: before-inactive-scan: Page shortage: -34665, inactive page surplus: 10000 2015 Oct 13 18:01:09: v_free_target: 20888, v_free_count: 55553, v_cache_count: 0. 2015 Oct 13 18:01:09: v_inactive_target: 31332. v_inactive_count: 367490. deficit: 0 dtrace: cpu clocks out of sync (18131462608939 < 18131468746340; offset: 6137401). Results may be incorrect! Assertion failed: (timestamp >= dtp->dt_last_timestamp), file /usr/src/cddl/lib/libdtrace/../../../cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c, line 3352. --- cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c index 39530cf..3b66c35 100644 --- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c +++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c @@ -3349,7 +3349,13 @@ dtrace_consume(dtrace_hdl_t *dtp, FILE *fp, break; timestamp = dt_buf_oldest(buf, dtp); - assert(timestamp >= dtp->dt_last_timestamp); + if (timestamp < dtp->dt_last_timestamp) { + warnx("cpu clocks out of sync " + "(%ju < %ju; offset: %ju). " + "Results may be incorrect!", + timestamp, dtp->dt_last_timestamp, + dtp->dt_last_timestamp - timestamp); + } dtp->dt_last_timestamp = timestamp; if (timestamp == buf->dtbd_timestamp) { -- 2.7.0 From 1027c381455f47f213b5fed88e698e98a668002c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 24 Oct 2015 12:58:42 +0200 Subject: [PATCH 189/213] src.opts.mk: Disable EXAMPLES so we don't have to care about license issues --- share/mk/src.opts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 71b480b..276ac03 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -76,7 +76,6 @@ __DEFAULT_YES_OPTIONS = \ ED_CRYPTO \ EE \ ELFTOOLCHAIN_BOOTSTRAP \ - EXAMPLES \ FDT \ FILE \ FINGER \ @@ -170,6 +169,7 @@ __DEFAULT_NO_OPTIONS += \ BSDINSTALL \ CCD \ CTM \ + EXAMPLES \ FLOPPY \ FMTREE \ FREEBSD_UPDATE \ -- 2.7.0 From 8dd36870d365cf942d61ec3d5187edb66e9ea0ef Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 23 Oct 2015 23:36:28 +0200 Subject: [PATCH 190/213] zfs: Do not advertise sha512, skein and edonr which aren't supported yet Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c index 20b54d8..57b82d0 100644 --- a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c +++ b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c @@ -240,12 +240,18 @@ zfs_prop_init(void) zprop_register_index(ZFS_PROP_CHECKSUM, "checksum", ZIO_CHECKSUM_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, - "on | off | fletcher2 | fletcher4 | sha256 | sha512 | " - "skein | edonr", "CHECKSUM", checksum_table); + "on | off | fletcher2 | fletcher4 | sha256" +#ifdef illumos + " | sha512 | skein | edonr" +#endif + , "CHECKSUM", checksum_table); zprop_register_index(ZFS_PROP_DEDUP, "dedup", ZIO_CHECKSUM_OFF, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, - "on | off | verify | sha256[,verify], sha512[,verify], " - "skein[,verify], edonr,verify", "DEDUP", dedup_table); + "on | off | verify | sha256[,verify]" +#ifdef illumos + ", sha512[,verify], skein[,verify], edonr,verify" +#endif + , "DEDUP", dedup_table); zprop_register_index(ZFS_PROP_COMPRESSION, "compression", ZIO_COMPRESS_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, -- 2.7.0 From b420ae22390eea5a59d7f2d95e7802f926257888 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 25 Oct 2015 19:49:47 +0100 Subject: [PATCH 191/213] ZFS dsl_scan_visitds(): Don't panic if a device disappears while scrubbing Prevents: Unread portion of the kernel message buffer: [4299] panic: solaris assert: dmu_objset_find_dp(dp, dp->dp_root_dir_obj, enqueue_clones_cb, &eca, (1<<1)) == 0 (0x6 == 0x0), file: /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c, line: 1130 [4299] cpuid = 1 [4299] KDB: stack backtrace: [4299] db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe00949ed310 [4299] vpanic() at vpanic+0x182/frame 0xfffffe00949ed390 [4299] panic() at panic+0x43/frame 0xfffffe00949ed3f0 [4299] zfs_kmem_alloc() at zfs_kmem_alloc/frame 0xfffffe00949ed440 [4299] dsl_scan_visitds() at dsl_scan_visitds+0x551/frame 0xfffffe00949ed570 [4299] dsl_scan_visit() at dsl_scan_visit+0x22e/frame 0xfffffe00949ed790 [4299] dsl_scan_sync() at dsl_scan_sync+0x9da/frame 0xfffffe00949ed920 [4299] spa_sync() at spa_sync+0x564/frame 0xfffffe00949eda90 [4299] txg_sync_thread() at txg_sync_thread+0x3f1/frame 0xfffffe00949edbb0 [4299] fork_exit() at fork_exit+0x9c/frame 0xfffffe00949edbf0 [4299] fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00949edbf0 [4299] --- trap 0, rip = 0, rsp = 0, rbp = 0 --- [4299] KDB: enter: panic Note that this is not the only place where dmu_objset_find_dp() failures lead to panics and ENXIO probably isn't the only return code we have to expect anyway. --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c index 4023985..aa85666 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c @@ -1122,12 +1122,14 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) scn->scn_phys.scn_queue_obj, dsl_dataset_phys(ds)->ds_creation_txg, tx)); } else { + int error; struct enqueue_clones_arg eca; eca.tx = tx; eca.originobj = ds->ds_object; - VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, - enqueue_clones_cb, &eca, DS_FIND_CHILDREN)); + error = dmu_objset_find_dp(dp, dp->dp_root_dir_obj, + enqueue_clones_cb, &eca, DS_FIND_CHILDREN); + VERIFY(error == 0 || error == ENXIO); } } -- 2.7.0 From 9d49dd4de2e36c37f30057dca73a5ef24dae2dd3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 27 Oct 2015 23:09:03 +0100 Subject: [PATCH 192/213] ZFS: Optionally let spa_sync() wait until at least one vdev is writable ... before calling dsl_pool_sync(). Significantly reduces the chances that a temporary loss of all vdevs results in a spa_namespace_lock deadlock that causes the pool to become inaccesible until the next reboot if the failmode is "wait" (default). The deadlock is caused by spa_sync() holding the required locks to fault or clear the pool while waiting for the sync operation to complete, thus preventing other parts of the system to get the pool in a usable state again. As a side effect of the deadlock, various zfs and zpool commands can lock up as well. Obtained from: ElectroBSD Deadlock example: 6 102135 zfskern txg_thread_enter mi_switch+0xde sleepq_wait+0x3a _cv_wait+0x1a6 zio_wait+0x89 dsl_pool_sync+0x12e spa_sync+0x51f txg_sync_thread+0x408 fork_exit+0x9a fork_trampoline+0xe 5985 101966 zpool - mi_switch+0xde sleepq_wait+0x3a _sx_xlock_hard+0x540 _sx_xlock+0x5d spa_get_errlog_size+0x3a spa_get_stats+0x116 zfs_ioc_pool_stats+0x4c zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 Xfast_syscall+0xfb 6863 101673 zpool - mi_switch+0xde sleepq_wait+0x3a _cv_wait+0x1a6 zio_wait+0x89 dbuf_read+0x464 dmu_buf_hold+0x9a zap_get_leaf_byblk+0x76 zap_deref_leaf+0xc6 fzap_cursor_retrieve+0x169 zap_cursor_retrieve+0x216 process_error_log+0xb1 spa_get_errlog+0xae zfs_ioc_error_log+0x70 zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 6885 100784 zpool - mi_switch+0xde sleepq_wait+0x3a _sx_xlock_hard+0x540 _sx_xlock+0x5d spa_get_errlog_size+0x3a spa_get_stats+0x116 zfs_ioc_pool_stats+0x4c zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 Xfast_syscall+0xfb See also: https://lists.freebsd.org/pipermail/freebsd-current/2014-September/052018.html Log excerpt with the sysctl enabled (single-disk pool): Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: g_eli_read_done() failed (error=5) label/prot1.eli[READ(offset=4654291968, length=9216)] Oct 28 12:27:03 r500 kernel: [415] da0 at umass-sim0 bus 0 scbus2 target 0 lun 0 Oct 28 12:27:03 r500 kernel: [415] da0: s/n 22B39DCC detached Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: Device label/prot1.eli destroyed. Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: Detached label/prot1.eli on last close. Oct 28 12:27:03 r500 kernel: [415] (da0:umass-sim0:0:0:0): Periph destroyed Oct 28 12:27:03 r500 ZFS: vdev is removed, pool_guid=2852801396507945726 vdev_guid=18325185749120546863 Oct 28 12:27:04 r500 kernel: [416] spa_sync: No writable vdev for prot1. Oct 28 12:27:05 r500 kernel: [417] spa_sync: No writable vdev for prot1. Oct 28 12:27:06 r500 kernel: [418] spa_sync: No writable vdev for prot1. Oct 28 12:27:07 r500 kernel: [419] spa_sync: No writable vdev for prot1. Oct 28 12:27:08 r500 kernel: [420] spa_sync: No writable vdev for prot1. Oct 28 12:27:09 r500 kernel: [420] umass0: on usbus1 Oct 28 12:27:09 r500 kernel: [420] umass0: SCSI over Bulk-Only; quirks = 0x4101 Oct 28 12:27:09 r500 kernel: [420] umass0:2:0: Attached to scbus2 Oct 28 12:27:09 r500 kernel: [420] da0 at umass-sim0 bus 0 scbus2 target 0 lun 0 Oct 28 12:27:09 r500 kernel: [420] da0: Removable Direct Access SCSI-2 device Oct 28 12:27:09 r500 kernel: [420] da0: Serial Number 22B39DCC Oct 28 12:27:09 r500 kernel: [420] da0: 40.000MB/s transfers Oct 28 12:27:09 r500 kernel: [420] da0: 7782MB (15937536 512 byte sectors) Oct 28 12:27:09 r500 kernel: [420] da0: quirks=0x2 Oct 28 12:27:09 r500 kernel: [421] GEOM_PART: integrity check failed (label/prot1, MBR) Oct 28 12:27:09 r500 kernel: [421] spa_sync: No writable vdev for prot1. Oct 28 12:27:10 r500 kernel: [422] spa_sync: No writable vdev for prot1. Oct 28 12:27:11 r500 kernel: [423] spa_sync: No writable vdev for prot1. Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Device label/prot1.eli created. Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Encryption: AES-XTS 256 Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Crypto: software Oct 28 12:27:12 r500 ZFS: vdev state changed, pool_guid=2852801396507945726 vdev_guid=18325185749120546863 Oct 28 12:27:12 r500 kernel: [424] spa_sync: vdev for prot1 became writable again. Device loss was triggered and fixed with: sudo usbconfig -d 1.2 reset && zogftw import prot1 && sudo zpool clear prot1 This commit is experimental! It should be a nop unless the vfs.zfs.spa_sync_wait_for_writable_vdev sysctl is set, though. Setting the sysctl merely shrinks the race window and thus does not completely solve the problem. The problem can still be reproduced with an artifical test case like: Shell 1: mdconfig -u 0 -f /dpool/scratch/test-vdev.img zpool create test /dev/md0 while sleep 1; do mdconfig -d -u 0 -o force && mdconfig -f /dpool/scratch/test-vdev.img && zpool clear test; done Shell 2: # Cause writes to the pool from another shell, for example # by creating datasets. Log excerpt (from test begin to deadlock): Oct 29 12:34:28 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:44:42 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:45:04 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:43 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:43 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:44 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:44 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:45 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:45 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:46 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:46 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:47 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:47 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:48 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:48 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:49 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:49 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:51 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:51 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:52 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:52 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:53 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:54 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:54 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:55 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:55 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:56 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:56 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:57 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:57 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:59 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:59 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:47:00 kendra kernel: g_dev_taste: make_dev_p() failed (gp->name=md0, error=17) With the deadman enabled, this will also cause: panic: I/O to pool 'test' appears to be hung on vdev guid 3080051161477470469 at '/dev/md0'. cpuid = 0 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe01136af870 vpanic() at vpanic+0x182/frame 0xfffffe01136af8f0 panic() at panic+0x43/frame 0xfffffe01136af950 vdev_deadman() at vdev_deadman+0x127/frame 0xfffffe01136af9a0 vdev_deadman() at vdev_deadman+0x40/frame 0xfffffe01136af9f0 spa_deadman() at spa_deadman+0x86/frame 0xfffffe01136afa20 softclock_call_cc() at softclock_call_cc+0x1a3/frame 0xfffffe01136afaf0 softclock() at softclock+0x94/frame 0xfffffe01136afb20 intr_event_execute_handlers() at intr_event_execute_handlers+0x1b6/frame 0xfffffe01136afb60 ithread_loop() at ithread_loop+0xa6/frame 0xfffffe01136afbb0 fork_exit() at fork_exit+0x9c/frame 0xfffffe01136afbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01136afbf0 With test's txg_sync_thread being the offender: (kgdb) tid 101874 [Switching to thread 819 (Thread 101874)]#0 sched_switch (td=0xfffff800513649a0, newtd=, flags=) at /usr/src/sys/kern/sched_ule.c:1969 1969 cpuid = PCPU_GET(cpuid); (kgdb) where #0 sched_switch (td=0xfffff800513649a0, newtd=, flags=) at /usr/src/sys/kern/sched_ule.c:1969 #1 0xffffffff805a3a18 in mi_switch (flags=260, newtd=0x0) at /usr/src/sys/kern/kern_synch.c:470 #2 0xffffffff805ea15a in sleepq_wait (wchan=0x0, pri=0) at /usr/src/sys/kern/subr_sleepqueue.c:631 #3 0xffffffff80530509 in _cv_wait (cvp=0xfffff8002678ea98, lock=0xfffff8002678ea78) at /usr/src/sys/kern/kern_condvar.c:139 #4 0xffffffff81930bbb in zio_wait (zio=) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:1535 #5 0xffffffff818e4871 in dsl_pool_sync (dp=0xfffff80047dfd000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c:540 #6 0xffffffff81903653 in spa_sync (spa=0xfffff8009dfe2000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6734 #7 0xffffffff8190ccfa in txg_sync_thread (arg=0xfffff80047dfd000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:517 #8 0xffffffff80556edc in fork_exit (callout=0xffffffff8190c970 , arg=0xfffff80047dfd000, frame=0xfffffe011c27bc00) at /usr/src/sys/kern/kern_fork.c:1011 #9 0xffffffff8085b91e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:609 #10 0x0000000000000000 in ?? () (kgdb) f 6 #6 0xffffffff81903653 in spa_sync (spa=0xfffff8009dfe2000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6734 (kgdb) p spa->spa_name $3 = 0xfffff8009dfe2000 "test" --- .../contrib/opensolaris/uts/common/fs/zfs/spa.c | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c index 9caed4d..7be8f17 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c @@ -6589,6 +6589,31 @@ spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) rrw_exit(&dp->dp_config_rwlock, FTAG); } + +/* + * Check if there's at least one (potentionally) writable vdev. + */ +int +spa_has_writable_vdev(spa_t *spa) { + vdev_t *rvd = spa->spa_root_vdev; + vdev_t *vd; + int i; + + for (i = 0; i < rvd->vdev_children; i++) { + vd = rvd->vdev_child[i]; + if (!vd->vdev_cant_write) + return 1; + } + return 0; +} + +static int spa_sync_wait_for_writable_vdev = 0; +SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_sync_wait_for_writable_vdev, CTLFLAG_RWTUN, + &spa_sync_wait_for_writable_vdev, 0, + "Let spa_sync() pause if no writable vdev is available. Experimental!"); + +static int spa_sync_deadlock_loops = 0; + /* * Sync the specified transaction group. New blocks may be dirtied as * part of the process, so we iterate until it converges. @@ -6684,6 +6709,29 @@ spa_sync(spa_t *spa, uint64_t txg) spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); spa_errlog_sync(spa, txg); + if (spa_sync_wait_for_writable_vdev) { + int waited_for_vdev = 0; + + while (!spa_has_writable_vdev(spa)) { + spa_config_exit(spa, SCL_CONFIG, FTAG); + + waited_for_vdev = 1; + printf("%s: No writable vdev for %s.\n", + __func__, spa_name(spa)); +#ifdef _KERNEL + pause("spa_sync", hz); +#else + sleep(1); +#endif + spa_sync_deadlock_loops++; + + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + } + if (waited_for_vdev) { + printf("%s: vdev for %s became writable again.\n", + __func__, spa_name(spa)); + } + } dsl_pool_sync(dp, txg); if (pass < zfs_sync_pass_deferred_free) { -- 2.7.0 From 75a2b65a651e7977ac1db4f27047e72e6052f293 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 18 Mar 2015 13:08:28 +0100 Subject: [PATCH 193/213] parse_mount(): Use vfs.mountroot.timeout for ZFS root pools as well Instead of trying to figure out the required vdevs to wait for, just call kernel_mount() until it works or the time is up. Fancier approaches are conceivable ... If the current approach is kept, it may make sense to add a flag to tell kernel_mount() not to free the mount args. As an alternative the retrying could be delegated to kernel_mount() itself. Obtained from: ElectroBSD --- sys/kern/vfs_mountroot.c | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c index cf24253..528af1d 100644 --- a/sys/kern/vfs_mountroot.c +++ b/sys/kern/vfs_mountroot.c @@ -704,7 +704,7 @@ parse_mount(char **conf) char *errmsg; struct mntarg *ma; char *dev, *fs, *opts, *tok; - int error; + int delay, error, timeout; error = parse_token(conf, &tok); if (error) @@ -745,15 +745,29 @@ parse_mount(char **conf) if (error != 0) goto out; - ma = NULL; - ma = mount_arg(ma, "fstype", fs, -1); - ma = mount_arg(ma, "fspath", "/", -1); - ma = mount_arg(ma, "from", dev, -1); - ma = mount_arg(ma, "errmsg", errmsg, ERRMSGL); - ma = mount_arg(ma, "ro", NULL, 0); - ma = parse_mountroot_options(ma, opts); - error = kernel_mount(ma, MNT_ROOTFS); + delay = hz / 10; + timeout = root_mount_timeout * hz; + do { + ma = NULL; + ma = mount_arg(ma, "fstype", fs, -1); + ma = mount_arg(ma, "fspath", "/", -1); + ma = mount_arg(ma, "from", dev, -1); + ma = mount_arg(ma, "errmsg", errmsg, ERRMSGL); + ma = mount_arg(ma, "ro", NULL, 0); + ma = parse_mountroot_options(ma, opts); + + error = kernel_mount(ma, MNT_ROOTFS); + if (strcmp(fs, "zfs") != 0) + break; + timeout -= delay; + if (timeout > 0 && error) { + pause("rmdev", delay); + printf("Mounting from %s:%s failed with error %d. " + "%d seconds left. Retrying.\n", fs, dev, error, + timeout / hz); + } + } while (timeout > 0 && error); out: if (error) { printf("Mounting from %s:%s failed with error %d", @@ -957,8 +971,12 @@ vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev) int delay, timeout; /* - * In case of ZFS and NFS we don't have a way to wait for - * specific device. + * For ZFS we can't simply wait for a specific device + * as we only know the pool name. To work around this, + * parse_mount() will retry the mount later on. + * + * While retrying for NFS could be implemented similarly + * it is currently not supported. */ if (strcmp(fs, "zfs") == 0 || strstr(fs, "nfs") != NULL || dev[0] == '\0') { -- 2.7.0 From cd577c551695e6bc5570b6300eead04c237a1076 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 1 Nov 2015 14:21:15 +0100 Subject: [PATCH 194/213] Use the mount retry timeout for NFS as well. Untested. --- sys/kern/vfs_mountroot.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c index 528af1d..4bf9b74 100644 --- a/sys/kern/vfs_mountroot.c +++ b/sys/kern/vfs_mountroot.c @@ -758,7 +758,7 @@ parse_mount(char **conf) ma = parse_mountroot_options(ma, opts); error = kernel_mount(ma, MNT_ROOTFS); - if (strcmp(fs, "zfs") != 0) + if (strcmp(fs, "zfs") != 0 || strstr(fs, "nfs") != NULL) break; timeout -= delay; if (timeout > 0 && error) { @@ -971,12 +971,9 @@ vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev) int delay, timeout; /* - * For ZFS we can't simply wait for a specific device + * For ZFS and NFS we can't simply wait for a specific device * as we only know the pool name. To work around this, * parse_mount() will retry the mount later on. - * - * While retrying for NFS could be implemented similarly - * it is currently not supported. */ if (strcmp(fs, "zfs") == 0 || strstr(fs, "nfs") != NULL || dev[0] == '\0') { -- 2.7.0 From 5cad0121694ee0da81d56f7041a5f16a3a992973 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 13:49:38 +0200 Subject: [PATCH 195/213] sys/modules: Don't (try to) build qlxbge if the user objects to binary blobs The module depends on source-less microcode in sys/dev/qlxgbe/ql_fw.c. Obtained from: ElectroBSD --- sys/modules/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index a834873..a4999a3 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -656,7 +656,9 @@ _ntb= ntb _pms= pms _qlxge= qlxge _qlxgb= qlxgb +.if ${MK_SOURCELESS_UCODE} != "no" _qlxgbe= qlxgbe +.endif _sfxge= sfxge .if ${MK_BHYVE} != "no" || defined(ALL_MODULES) -- 2.7.0 From d62fc86d4fac0c534be74b60fd4f343a65e3a5a0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 12:52:29 +0200 Subject: [PATCH 196/213] sys/modules: Don't (try to) build bxe if the user objects to binary blobs The module contains source-less microcode: sys/dev/bxe/5771[0-2]_init_values.c Obtained from: ElectroBSD --- sys/modules/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index a4999a3..ebcfd2a 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -521,7 +521,9 @@ _agp= agp _an= an _aout= aout _bktr= bktr +.if ${MK_SOURCELESS_UCODE} != "no" _bxe= bxe +.endif _cardbus= cardbus _cbb= cbb _cpuctl= cpuctl -- 2.7.0 From d7b519f38dbc8769871caaf436dacf7b1a9d8152 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 26 Sep 2015 20:49:09 +0200 Subject: [PATCH 197/213] sys/modules: Don't (try to) build otusfw if the user objects to binary blobs The modules contains source-less and non-free microcode: sys/contrib/dev/otus/ Obtained from: ElectroBSD --- sys/modules/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index ebcfd2a..78efaf8 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -271,7 +271,7 @@ SUBDIR= \ ${_nxge} \ oce \ otus \ - otusfw \ + ${_otusfw} \ ow \ ${_padlock} \ ${_padlock_rng} \ @@ -478,6 +478,7 @@ _bce= bce _fxp= fxp _ispfw= ispfw _mwlfw= mwlfw +_otusfw= otusfw _ralfw= ralfw _rtwnfw= rtwnfw _sf= sf -- 2.7.0 From 13082cbfb74dafce0749cddaf09d657ea2fe78aa Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 27 Nov 2015 00:20:59 +0100 Subject: [PATCH 198/213] etc: Register uid and gid for Tor ... so the user doesn't have to do it manually on the host system after installing Tor in a jail. --- etc/group | 1 + etc/master.passwd | 1 + 2 files changed, 2 insertions(+) diff --git a/etc/group b/etc/group index 3136230..49eccdc 100644 --- a/etc/group +++ b/etc/group @@ -30,6 +30,7 @@ network:*:69: audit:*:77: www:*:80: _ypldap:*:160: +_tor:*:256: hast:*:845: ggatec:*:846: ggated:*:847: diff --git a/etc/master.passwd b/etc/master.passwd index 44a1609..65e75a5 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -23,6 +23,7 @@ pop:*:68:6::0:0:Post Office Owner:/nonexistent:/usr/sbin/nologin auditdistd:*:78:77::0:0:Auditdistd unprivileged user:/var/empty:/usr/sbin/nologin www:*:80:80::0:0:World Wide Web Owner:/nonexistent:/usr/sbin/nologin _ypldap:*:160:160::0:0:YP Ldap unprivileged user:/var/empty:/usr/sbin/nologin +_tor:*:256:256::0:0:Onion delivery agent:/nonexistent:/usr/sbin/nologin hast:*:845:845::0:0:HAST unprivileged user:/var/empty:/usr/sbin/nologin ggatec:*:846:846::0:0:ggatec unprivileged user:/var/empty:/usr/sbin/nologin ggated:*:847:847::0:0:ggated unprivileged user:/var/empty:/usr/sbin/nologin -- 2.7.0 From 5776712a86c53e956bebc9ee457a33638a077d92 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 16 Dec 2015 12:28:43 +0100 Subject: [PATCH 199/213] fork_findpid(): Declare p static to help diagnosing panics Like this one: Fatal trap 12: page fault while in kernel mode cpuid = 1; apic id = 04 fault virtual address = 0x618b00a8 fault code = supervisor read data, page not present instruction pointer = 0x20:0xffffffff80909158 stack pointer = 0x28:0xfffffe011e03b940 frame pointer = 0x28:0xfffffe011e03b960 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 71325 (sh) trap number = 12 panic: page fault cpuid = 1 KDB: stack backtrace: [...] Uptime: 13d20h43m20s [...] (kgdb) where #0 doadump (textdump=1) at pcpu.h:221 #1 0xffffffff8094a923 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:364 #2 0xffffffff8094ae8b in vpanic (fmt=, ap=) at /usr/src/sys/kern/kern_shutdown.c:757 #3 0xffffffff8094acc3 in panic (fmt=0x0) at /usr/src/sys/kern/kern_shutdown.c:688 #4 0xffffffff80c2fbb1 in trap_fatal (frame=, eva=) at /usr/src/sys/amd64/amd64/trap.c:834 #5 0xffffffff80c2fda4 in trap_pfault (frame=0xfffffe011e03b890, usermode=) at /usr/src/sys/amd64/amd64/trap.c:684 #6 0xffffffff80c2f55e in trap (frame=0xfffffe011e03b890) at /usr/src/sys/amd64/amd64/trap.c:435 #7 0xffffffff80c120a7 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:234 #8 0xffffffff80909158 in fork_findpid (flags=) at /usr/src/sys/kern/kern_fork.c:281 #9 0xffffffff80907225 in do_fork (td=0xfffff8009db9a9a0, flags=20, p2=0xfffff8009dbe1a90, td2=0xfffff800aa6884d0, vm2=0xfffff800a9eee000, pdflags=0) at /usr/src/sys/kern/kern_fork.c:385 #10 0xffffffff80906c08 in fork1 (td=0xfffff8009db9a9a0, flags=20, pages=, procp=0xfffffe011e03bac0, procdescp=0x0, pdflags=99999, fcaps=) at /usr/src/sys/kern/kern_fork.c:937 #11 0xffffffff809066ca in sys_fork (td=0xfffff8009db9a9a0, uap=) at /usr/src/sys/kern/kern_fork.c:108 #12 0xffffffff80c3054b in amd64_syscall (td=0xfffff8009db9a9a0, traced=0) at subr_syscall.c:140 #13 0xffffffff80c1238b in Xfast_syscall () at /usr/src/sys/amd64/amd64/exception.S:394 #14 0x00000008009257aa in ?? () Previous frame inner to this frame (corrupt stack?) Current language: auto; currently minimal (kgdb) f 8 #8 0xffffffff80909158 in fork_findpid (flags=) at /usr/src/sys/kern/kern_fork.c:281 warning: Source file is more recent than executable. 281 (p->p_pgrp != NULL && (kgdb) l - 271 * id is kept reserved only while there is a 272 * non-reaped process in the subtree, so amount of 273 * reserved pids is limited by process limit times 274 * two. 275 */ 276 p = LIST_FIRST(&allproc); 277 again: 278 for (; p != NULL; p = LIST_NEXT(p, p_list)) { 279 while (p->p_pid == trypid || 280 p->p_reapsubtree == trypid || (kgdb) l 281 (p->p_pgrp != NULL && 282 (p->p_pgrp->pg_id == trypid || 283 (p->p_session != NULL && 284 p->p_session->s_sid == trypid)))) { 285 trypid++; 286 if (trypid >= pidchecked) 287 goto retry; 288 } 289 if (p->p_pid > trypid && pidchecked > p->p_pid) 290 pidchecked = p->p_pid; (kgdb) p p No symbol "p" in current context. --- sys/kern/kern_fork.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index a6f30e2..2d7cdbd 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -216,7 +216,7 @@ SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW, static __noinline int fork_findpid(int flags) { - struct proc *p; + static struct proc *p; int trypid; static int pidchecked = 0; -- 2.7.0 From 031ebc0859b02b8ffdd50548edc2b6fbba1bc8cb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 25 Dec 2015 15:24:18 +0100 Subject: [PATCH 200/213] reproduce.sh: Add -p option to change the prefix for the source and object directories This could be useful when building ElectroBSD as port. --- reproduce.sh | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index dac0e34..4fbb77c 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -33,8 +33,14 @@ EPOCH=__EPOCH__ # This is just a suggestion, feel free to overwrite it with the -j option. MAX_MAKE_JOBS="${MAX_MAKE_JOBS-4}" -# Currently hardcoded. -SRC_DIR=/usr/src +# Prefix to use for SRC_DIR and MAKEOBJDIRPREFIX +DIRECTORY_PREFIX="${DIRECTORY_PREFIX-/}" + +# Currently somewhat hardcoded. +SRC_DIR="${DIRECTORY_PREFIX}usr/src" +MAKEOBJDIRPREFIX="${DIRECTORY_PREFIX}usr/obj" + +# Config file location when -f isn't specified OPTIONAL_CONFIG_FILE="${SRC_DIR}/reproduce.conf" # Make sure we respawn with the same script, even if it is located @@ -62,6 +68,8 @@ reproduce_all_the_things() { export KERNFAST=1 export NO_CLEAN=1 fi + export MAKEOBJDIRPREFIX + announce_status "MAKEOBJDIRPREFIX is set to ${MAKEOBJDIRPREFIX}" announce_status "Starting to build the kernel" make buildkernel || return 1 @@ -70,7 +78,7 @@ reproduce_all_the_things() { make -j${MAX_MAKE_JOBS} buildworld || return 1 # Make sure obj files aren't dumped in ${SRC_DIR} - mkdir -p "/usr/obj${SRC_DIR}/release" || return 1 + mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 if ! "${RESUME_BUILD}"; then announce_status "Starting to clean the release dir" @@ -105,6 +113,7 @@ respawn_with_clean_environment() { exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ + DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" \ REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" } @@ -118,7 +127,7 @@ main() { dry_run=false config_file="${OPTIONAL_CONFIG_FILE}" - args=$(getopt af:j:nr $*) + args=$(getopt af:j:npr $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -148,6 +157,17 @@ main() { dry_run=true shift ;; + -p) + shift + DIRECTORY_PREFIX="${1}" + shift; + if [ ! -d "${DIRECTORY_PREFIX}" ]; then + echo "Directory ${DIRECTORY_PREFIX} specified with -p does not exist" + exit 2 + fi + # This is only needed for the cd below + SRC_DIR="${DIRECTORY_PREFIX}${SRC_DIR}" + ;; -r) shift RESUME_BUILD=true -- 2.7.0 From fdcf52d377d691f4adabc4fd02510be984304dd3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 7 Jan 2016 16:58:50 +0100 Subject: [PATCH 201/213] sys/sys/copyright.h: Update copyright --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index a6121f7..96f4ffe 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -30,7 +30,7 @@ /* Add a FreeBSD vendor copyright here */ #define COPYRIGHT_Vendor \ - "Copyright (c) 2010-2015 Fabian Keil - IT-Beratung und Polizei-Erziehung\n" + "Copyright (c) 2010-2016 Fabian Keil - IT-Beratung und Polizei-Erziehung\n" /* FreeBSD */ #define COPYRIGHT_FreeBSD \ -- 2.7.0 From ed1c6cc1cbfc145ee5963f94e220b60e9917e4f4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 8 Jan 2016 13:30:21 +0100 Subject: [PATCH 202/213] ZFS ARC: Sync with FreeBSD bug #187594 (ZFS ARC behavior problem and fix) Main patch author: Karl Denninger PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594 PATCH URL: https://bz-attachments.freebsd.org/attachment.cgi?id=164051&action=diff&format=raw&headers=1 --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 62 +++++++++++++++++++++- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++- .../contrib/opensolaris/uts/common/fs/zfs/zio.c | 3 ++ 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index b41153f..550a3a7 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -242,6 +242,14 @@ int zfs_arc_p_min_shift = 0; int zfs_disable_dup_eviction = 0; uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ u_int zfs_arc_free_target = 0; +u_int zfs_arc_wakeup_pager = 0; +u_int zfs_arc_wakeup_delay = 500; + +#define WAKE_PAGER +#ifdef WAKE_PAGER +#define WAKE_PAGER_CONSTANT 10 / 9 /* Pager wakeup threshold */ +static int arc_init_done = 0; /* We know arc_warm is valid */ +#endif /* WAKE_PAGER */ static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS); static int sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS); @@ -252,6 +260,9 @@ arc_free_target_init(void *unused __unused) { zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); +#ifdef WAKE_PAGER + zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; +#endif /* WAKE_PAGER */ } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -273,6 +284,13 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW, SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, &zfs_dynamic_write_buffer, 0, "Dynamically restrict dirty data when memory is low"); +#ifdef WAKE_PAGER +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_pager, CTLFLAG_RWTUN, + &zfs_arc_wakeup_pager, 0, "Wake VM below this number of pages"); +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_delay, CTLFLAG_RWTUN, + &zfs_arc_wakeup_delay, 0, "May wake up VM once this number of MS"); +#endif /* WAKE_PAGER */ + /* * We don't have a tunable for arc_free_target due to the dependency on * pagedaemon initialisation. @@ -299,6 +317,9 @@ sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS) return (EINVAL); zfs_arc_free_target = val; +#ifdef WAKE_PAGER + zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; +#endif /* WAKE_PAGER */ return (0); } @@ -3390,6 +3411,11 @@ int64_t arc_pages_pp_reserve = 64; int64_t arc_swapfs_reserve = 64; /* + * Declare file-local static for event processor bypass + */ +static unsigned int arc_no_wake_event = 0; + +/* * Return the amount of memory that can be consumed before reclaim will be * needed. Positive if there is sufficient free memory, negative indicates * the amount of memory that needs to be freed up. @@ -3402,6 +3428,10 @@ arc_available_memory(void) free_memory_reason_t r = FMR_UNKNOWN; #ifdef _KERNEL +#ifdef WAKE_PAGER + sbintime_t now; + static sbintime_t last_pagedaemon_wake = 0; +#endif /* WAKE_PAGER */ if (needfree > 0) { n = PAGESIZE * (-needfree); if (n < lowest) { @@ -3420,6 +3450,26 @@ arc_available_memory(void) r = FMR_LOTSFREE; } +#ifdef WAKE_PAGER +/* + * If memory is less than the ARC wakeup threshold and time has expired since + * the last time we woke the pager... Do not execute until the ARC warms up. + */ + if ((arc_init_done) && + (((int64_t) freemem - zfs_arc_wakeup_pager) < 0) && + (arc_warm == B_TRUE) + ) { + now = getsbinuptime(); + if ((now - last_pagedaemon_wake) / SBT_1MS > zfs_arc_wakeup_delay) { + last_pagedaemon_wake = now; + arc_no_wake_event++; /* Set bypass flag for ARC */ + DTRACE_PROBE(arc__wake_pagedaemon); + pagedaemon_wakeup(); /* Wake the pager */ + } + } + +#endif /* WAKE_PAGER */ + #ifdef illumos /* * check that we're out of range of the pageout scanner. It starts to @@ -3528,6 +3578,7 @@ arc_available_memory(void) last_free_memory = lowest; last_free_reason = r; DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r); + return (lowest); } @@ -3558,9 +3609,10 @@ extern kmem_cache_t *range_seg_cache; * experimenting. */ static unsigned int arc_reaping_in_progress = 0; +static unsigned int arc_pagedaemon_ignore = 0; static sbintime_t last_reaping = 0; -static void __noinline +static __noinline void reap_arc_caches(void) { size_t i; @@ -5337,7 +5389,10 @@ static eventhandler_tag arc_event_lowmem = NULL; static void arc_lowmem(void *arg __unused, int howto __unused) { - + if (arc_no_wake_event) { /* Don't do it if we woke the pager */ + arc_no_wake_event = 0; /* Just clear the flag */ + return; + } mutex_enter(&arc_reclaim_lock); /* XXX: Memory deficit should be passed as argument. */ needfree = btoc(arc_c >> arc_shrink_shift); @@ -5595,6 +5650,9 @@ arc_init(void) printf(" in /boot/loader.conf.\n"); } #endif +#ifdef WAKE_PAGER + arc_init_done++; /* For anyone who wants to know */ +#endif /* WAKE_PAGER */ } void diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 6169a6d..70f2a0e 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1077,8 +1077,13 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) ASSERT3U(dirty, <, zfs_dirty_data_max_internal); now = gethrtime(); - min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + if (dirty >= zfs_dirty_data_max_internal) {/* No scaling if overcommitted */ + min_tx_time = zfs_delay_scale * + (dirty - delay_min_bytes); + } else { + min_tx_time = zfs_delay_scale * + (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + } if (now > tx->tx_start + min_tx_time) return; diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c index 089c314..3735491 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c @@ -43,6 +43,9 @@ SYSCTL_DECL(_vfs_zfs); SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); +/* KD 2015-07-15 Change class to "int" from "static int" as we reference + * this as an extern elsewhere + */ #if defined(__amd64__) int zio_use_uma = 1; #else -- 2.7.0 From 2d84072ca400360c9dc25f369c1fd85fed684812 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 Jan 2016 00:37:52 +0100 Subject: [PATCH 203/213] crontab: Default to not send out (most likely unencrypted) mails --- etc/crontab | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/crontab b/etc/crontab index e1e6e88..22204f8 100644 --- a/etc/crontab +++ b/etc/crontab @@ -1,9 +1,8 @@ -# /etc/crontab - root's crontab for FreeBSD -# -# $FreeBSD$ +# /etc/crontab - root's crontab for ElectroBSD # SHELL=/bin/sh PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin +MAILTO="" # #minute hour mday month wday who command # -- 2.7.0 From 76e5bb6fccf14dd944f4d88d2a77c9cbf10c2743 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 17:07:24 +0200 Subject: [PATCH 204/213] sys/cddl: Allow to modify the ZFS deadman sysctls after the system is up There does not seem to be a technical reason why it shouldn't be done and being able to temporarily disable the deadman is useful when importing a pool that is backed by remote storage (such as ggated running as onion service). Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c index d5764bc..ce30944 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c @@ -324,7 +324,7 @@ boolean_t zfs_free_leak_on_eio = B_FALSE; * in a system panic. */ uint64_t zfs_deadman_synctime_ms = 1000000ULL; -SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RDTUN, +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RWTUN, &zfs_deadman_synctime_ms, 0, "Stalled ZFS I/O expiration time in milliseconds"); @@ -333,7 +333,7 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RDTUN, * for hung I/O. */ uint64_t zfs_deadman_checktime_ms = 5000ULL; -SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RDTUN, +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RWTUN, &zfs_deadman_checktime_ms, 0, "Period of checks for stalled ZFS I/O in milliseconds"); @@ -342,7 +342,7 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RDTUN, * zfs_deadman_init() */ int zfs_deadman_enabled = -1; -SYSCTL_INT(_vfs_zfs, OID_AUTO, deadman_enabled, CTLFLAG_RDTUN, +SYSCTL_INT(_vfs_zfs, OID_AUTO, deadman_enabled, CTLFLAG_RWTUN, &zfs_deadman_enabled, 0, "Kernel panic on stalled ZFS I/O"); /* -- 2.7.0 From 62cca9881bb11e3e573d552d9e103c0b448e8a13 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 16 Jan 2016 15:24:51 +0100 Subject: [PATCH 205/213] Detach unreproducible uzip tests (added in r293821) that aren't relevant for ElectroBSD --- tests/sys/geom/class/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sys/geom/class/Makefile b/tests/sys/geom/class/Makefile index c705616..7a5d077 100644 --- a/tests/sys/geom/class/Makefile +++ b/tests/sys/geom/class/Makefile @@ -12,7 +12,8 @@ TESTS_SUBDIRS+= nop TESTS_SUBDIRS+= raid3 TESTS_SUBDIRS+= shsec TESTS_SUBDIRS+= stripe -TESTS_SUBDIRS+= uzip +# Currently not reproducible +#TESTS_SUBDIRS+= uzip BINDIR= ${TESTSDIR} -- 2.7.0 From c16b4e20f7211a01783043e1ff9fb5eded506841 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 29 May 2015 10:46:06 +0200 Subject: [PATCH 206/213] Import cloudiatr 2016-01-17-16414bb --- usr.sbin/Makefile | 1 + usr.sbin/cloudiatr/Makefile | 3 + usr.sbin/cloudiatr/cloudiatr | 1195 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1199 insertions(+) create mode 100644 usr.sbin/cloudiatr/Makefile create mode 100755 usr.sbin/cloudiatr/cloudiatr diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index 2003b86..a414ebf 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -8,6 +8,7 @@ SUBDIR= adduser \ binmiscctl \ bsdconfig \ camdd \ + cloudiatr \ cdcontrol \ chkgrp \ chown \ diff --git a/usr.sbin/cloudiatr/Makefile b/usr.sbin/cloudiatr/Makefile new file mode 100644 index 0000000..34ada85 --- /dev/null +++ b/usr.sbin/cloudiatr/Makefile @@ -0,0 +1,3 @@ +SCRIPTS= cloudiatr + +.include diff --git a/usr.sbin/cloudiatr/cloudiatr b/usr.sbin/cloudiatr/cloudiatr new file mode 100755 index 0000000..8029c0b --- /dev/null +++ b/usr.sbin/cloudiatr/cloudiatr @@ -0,0 +1,1195 @@ +#!/bin/sh + +########################################################################### +# cloudiatr +# +# Buzzword-compliant remote OS eviction tool. For details see: +# https://www.fabiankeil.de/gehacktes/cloudiatr/ +# +# Copyright (c) 2014-2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# It's important that this function is called before any other +# function except cloudiatr_main(), otherwise fatal errors may +# not be caught. +cloudiatr_init() { + local mode="${1}" + + set -e + cloudiatr_init_globals + + cloudiatr_load_config_file "${CLOUDIATR_CONFIG_FILE}" + + cloudiatr_check_config +} + +cloudiatr_fyi() { + local message="${*}" + + echo "cloudiatr: $message" +} + +cloudiatr_wtf() { + local complaints="${*}" + if [ -z "${complaints}" ]; then + complaints="cloudiatr_wtf(): No complaints?" + fi + cloudiatr_fyi "${complaints}" 1>&2 + return 1 +} + +cloudiatr_check_config() { + local v_flag \ + mandatory_variable optional_variable value fail + + v_flag="${1}" + fail=0 + + for mandatory_variable in ${CLOUDIA_MANDATORY_VARIABLES}; do + value="$(eval 'echo $'"${mandatory_variable}")" + if [ -z "${value}" ]; then + cloudiatr_wtf "Fatal error: ${mandatory_variable} is unset" + fail=1 + elif [ "${v_flag}" = "-v" ]; then + echo "${mandatory_variable}='${value}'" + fi + done + if [ "${v_flag}" = "-v" ]; then + for optional_variable in ${CLOUDIA_OPTIONAL_VARIABLES}; do + value="$(eval 'echo $'"${optional_variable}")" + echo "${optional_variable}='${value}'" + done + fi + return $fail +} + +cloudiatr_show_config() { + cloudiatr_check_config -v +} + +cloudiatr_load_config_file() { + local config_file="${1}" + + if [ -f "${config_file}" ]; then + . "${config_file}" + return 0 + fi + cloudiatr_wtf "Config file ${config_file} does not exist. You can use '$0 -f path/to/file ...' to specify a different one" + return 1 +} + +cloudiatr_init_globals() { + + CLOUDIATR_VERSION="2016-01-17-16414bb" + + CLOUDIATR_NEW_SYSTEM_DIR=/cloudiatr + # Only needs to be enough for a stripped-down bootfs + CLOUDIATR_BPOOL_PARTITION_SIZE=200M + # Has to be enough for the rest of the OS including the "permanent" + # /boot that is only used to (re)populate the bootfs on the bpool. + CLOUDIATR_RPOOL_PARTITION_SIZE=4G + CLOUDIATR_SWAP_PARTITION_SIZE=4G + + # Set to true to use the existing partition layout. + # + # Only expected to work if the layout was created by a previous + # cloudiatr run. Partitions 2, 3 and 4 will be overwritten, + # partition 1 is expected to contain working bootcode. + CLOUDIATR_REUSE_GPART_SETUP=false + + CLOUDIATR_BPOOL_NAME="bpool" + CLOUDIATR_RPOOL_NAME="rpool" + + # Note that DEFAULT is a fallback documented in rc.conf(5). + # Not changing this variable to the actual network interface + # is likely to cause problems if there are more than one nics + # and you aren't using DHCP for all of them. + CLOUDIATR_NIC="DEFAULT" + + # Will be created + CLOUDIATR_RPOOL_KEY_NAME="${CLOUDIATR_RPOOL_NAME}.key" + CLOUDIATR_RPOOL_KEY="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_KEY_NAME}" + + # If CLOUDIATR_DIST_IMAGE is set, an image containing the dist tarballs + # has to be put in place by the user before the eviction. If it's unset, + # CLOUDIATR_DIST_DIR has to be populated before cloudiatr is executed. + CLOUDIATR_DIST_IMAGE="" + CLOUDIATR_DIST_IMAGE_SHA256="" + CLOUDIATR_DIST_DIR="/usr/electrobsd-dist/" + + # Whether or not the distribution tarballs should be copied to the + # newly installed system (for example to reuse them when setting + # up jails). + CLOUDIATR_SAVE_DIST_DIR="false" + + # Default to using all the detected ada(4) devices + CLOUDIATR_DISKS="$(cloudiatr_autodetect_disks)" + + # Changing these should only be necessary if there's more + # than one disk and you don't want to a mirror. + CLOUDIATR_BPOOL_LAYOUT="default" + CLOUDIATR_RPOOL_LAYOUT="default" + + CLOUDIATR_GELI_KEY_LENGTH=256 + CLOUDIATR_GELI_EALGO=AES-XTS + + CLOUDIATR_SSHD_HOST_KEY_ALGORITHMS="rsa ecdsa ed25519" + + # Set to 'true' to ingore some safety-checks and increase the potential damage. + # Includes "geli kill -a" which is not limited to the disks specified above. + # Do not enable this unless the system that is being evicted doesn't contain + # any data you care about. + CLOUDIATR_MURDER_DEATH_KILL_REQUESTED=false + + # Set to 'true' to skip the image checksum check. + # "It may be insecure, but look how fast it is!" + CLOUDIATR_CHECKSUM_SMECKSUM=false + + # The config file is sourced and may overwrite any of the values above + # and most functions in this file (zogftw-style). + CLOUDIATR_CONFIG_FILE="${CLOUDIATR_CONFIG_FILE=/root/cloudiatr.conf}" + + # A user that should be created and allowed to "su" on the new system. + CLOUDIATR_NEW_USER="cloudiatr" + + # This password is used for both root and CLOUDIATR_NEW_USER. + # + # Note that the created system will not accept root logins through + # ssh (FreeBSD default). CLOUDIATR_NEW_USER may use ssh, but has + # to use public key authentication. + CLOUDIATR_INITIAL_PASSWORD="${CLOUDIATR_NEW_USER}" + + # Local timezone. For details see tzsetup(8). + CLOUDIATR_TIME_ZONE="Europe/Berlin" + + # When set to true, cloudiatr will execute ntpdate at installtime. + # The server(s) being used depend on the install distfiles. + CLOUDIATR_USE_NTPDATE="false" + + # Any alignment should work, 1M is often recommended to prevent + # write-amplification which can result in performance degradation. + # It can also be advantageous for trimming SSDs. + CLOUDIATR_GPART_ALIGNMENT="1M" + + # Optional keyboard map for the virtual console. + # For details see kbdmap. + CLOUDIATR_KBDMAP="de.kbd" + + # Additional distributions to extract. Example: src, lib32 + CLOUDIATR_EXTRA_DISTRIBUTIONS="" + + # Value for rc.conf's rether_enable entry which controls + # whether or not MAC addresses are randomized (on ElectroBSD). + CLOUDIATR_RETHER_ENABLE="NO" + + # Set to true to not bother the user about with questions. + : "${CLOUDIATR_DONT_ASK_JUST_KISS=false}" + + # Set to true (default) to use the added swap partitions right + # after creating them. This allows installations on systems that + # have insufficient memory (512 MB, for example) and no previously + # configured swap devices. + # + # While this option is not expected to cause problems, if you + # are absolutely sure that enough memory is available you can + # disable the behaviour by setting the variable to "false". + CLOUDIATR_USE_SWAP_WHILE_INSTALLING="true" + + # If these variables aren't set to some value, cloudiatr will abort. + # Sane values are a good idea but not mandatory. + CLOUDIA_MANDATORY_VARIABLES="\ + CLOUDIATR_BPOOL_LAYOUT \ + CLOUDIATR_BPOOL_NAME \ + CLOUDIATR_BPOOL_PARTITION_SIZE \ + CLOUDIATR_CHECKSUM_SMECKSUM \ + CLOUDIATR_CONFIG_FILE \ + CLOUDIATR_DISKS \ + CLOUDIATR_DIST_DIR \ + CLOUDIATR_DONT_ASK_JUST_KISS \ + CLOUDIATR_GELI_KEY_LENGTH \ + CLOUDIATR_GELI_EALGO \ + CLOUDIATR_GPART_ALIGNMENT \ + CLOUDIATR_HOSTNAME \ + CLOUDIATR_INITIAL_PASSWORD \ + CLOUDIATR_MURDER_DEATH_KILL_REQUESTED \ + CLOUDIATR_NEW_SYSTEM_DIR \ + CLOUDIATR_NEW_USER \ + CLOUDIATR_NIC \ + CLOUDIATR_USE_SWAP_WHILE_INSTALLING \ + CLOUDIATR_RETHER_ENABLE \ + CLOUDIATR_REUSE_GPART_SETUP \ + CLOUDIATR_RPOOL_KEY \ + CLOUDIATR_RPOOL_KEY_NAME \ + CLOUDIATR_RPOOL_LAYOUT \ + CLOUDIATR_RPOOL_NAME \ + CLOUDIATR_RPOOL_PARTITION_SIZE \ + CLOUDIATR_SAVE_DIST_DIR \ + CLOUDIATR_SWAP_PARTITION_SIZE \ + CLOUDIATR_TIME_ZONE \ + CLOUDIATR_USE_NTPDATE \ + CLOUDIATR_VERSION \ + " + + # These variables are allowed to be unset + CLOUDIA_OPTIONAL_VARIABLES="\ + CLOUDIATR_DEFAULTROUTER \ + CLOUDIATR_DIST_IMAGE \ + CLOUDIATR_DIST_IMAGE_SHA256 \ + CLOUDIATR_EXTRA_DISTRIBUTIONS \ + CLOUDIATR_IP_ADDRESS \ + CLOUDIATR_KBDMAP \ + CLOUDIATR_NETMASK \ + " +} + +cloudiatr_gpart_disk() { + local disk="${1}" \ + disk_name + + disk_name="${disk##*/}" + + gpart create -s gpt "${disk}" + + gpart add -s 128 -t freebsd-boot "${disk}" + gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 "${disk}" + + gpart add -s "${CLOUDIATR_BPOOL_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "${CLOUDIATR_BPOOL_NAME}-${disk_name}" -t freebsd-zfs "${disk}" + gpart add -s "${CLOUDIATR_RPOOL_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "${CLOUDIATR_RPOOL_NAME}-${disk_name}" -t freebsd-zfs "${disk}" + gpart add -s "${CLOUDIATR_SWAP_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "swap-${disk_name}" -t freebsd-swap "${disk}" + # Reserve what's left for the data pool + gpart add -l "dpool-${disk_name}" -a "${CLOUDIATR_GPART_ALIGNMENT}" -t freebsd-zfs "${disk}" +} + +cloudiatr_gpart_setup() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Cleaning partition tables (if there are any) ..." + for d in $disks; do + gpart destroy -F "${d}" 2>/dev/null || true + done + + cloudiatr_fyi "Partitioning disks ..." + for d in $disks; do + cloudiatr_gpart_disk "${d}" + done +} + +# Use the swap partitions on the given disks while cloudiatr is running. +# This allows to install on a system with 512MB RAM or less and no swap space. +cloudiatr_enable_swap() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Using created swap space while installing ..." + for d in $disks; do + geli onetime -d "${d}p4" || return 1 + swapon "${d}p4.eli" || return 1 + done +} + +cloudiatr_disable_swap() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Trying to disable prevously added swap space ..." + for d in $disks; do + swapoff "${d}p4.eli" || return 1 + done +} + +cloudiatr_get_geoms() { + local postfix="${1}" \ + d geoms + + for d in ${CLOUDIATR_DISKS}; do + geom_partition="${d}${postfix}" + geoms="${geoms} ${geom_partition}" + done + echo "${geoms## }" +} + +cloudiatr_get_gpart_labels() { + local postfix="${1}" + + for d in ${CLOUDIATR_DISKS}; do + d="${d##/dev/}" + gpart show -l -p "${d}" 2>/dev/null | awk '$3 == "'"${d}${postfix}"'" {printf "%s ", $4}' + done + echo +} + +# Depends on geli being already setup +cloudiatr_create_rpool() { + local \ + rpool_elis pool_layout + + rpool_elis="$(cloudiatr_get_geoms p3.eli)" + pool_layout="${CLOUDIATR_RPOOL_LAYOUT}" + + if [ "${pool_layout}" = "default" ]; then + pool_layout="$(cloudiatr_get_default_pool_layout)" + fi + + cloudiatr_fyi "Creating root pool '${CLOUDIATR_RPOOL_NAME}' on ${rpool_elis}. Pool layout: ${pool_layout}" + + zpool create -o version=28 -o failmode=continue \ + -O compression=lzjb -O checksum=sha256 \ + "${CLOUDIATR_RPOOL_NAME}" ${pool_layout##single-disk} $rpool_elis + + zfs set mountpoint="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}" "${CLOUDIATR_RPOOL_NAME}" + + zfs create "${CLOUDIATR_RPOOL_NAME}/boot" + # We currently use no dedicated dataset for /etc as the kernel expects parts of it + # to be available once the rootfs has been mounted. Having two /etc's can be a bit + # of a hassle on updates and thus doesn't seem like a good default. + #zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/etc" + zfs create "${CLOUDIATR_RPOOL_NAME}/home" + zfs create "${CLOUDIATR_RPOOL_NAME}/home/${CLOUDIATR_NEW_USER}" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/tmp" + zfs create "${CLOUDIATR_RPOOL_NAME}/usr" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/src" + zfs create "${CLOUDIATR_RPOOL_NAME}/var" + zfs create "${CLOUDIATR_RPOOL_NAME}/usr/local" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/local/etc" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports" + zfs create -o compression=off -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports/distfiles" + zfs create -o compression=off -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports/packages" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/crash" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/db" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/db/pkg" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/empty" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/log" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/mail" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/run" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/tmp" + + chmod 0750 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/crash" + chgrp mail "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/mail" + chmod 0775 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/mail" + chmod 0555 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/empty" + chflags schg,nouarch "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/empty" + chmod 1777 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/tmp" + chmod 1777 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/tmp" + + zfs set readonly=on "${CLOUDIATR_RPOOL_NAME}/var/empty" +} + +cloudiatr_kernel_module_is_loaded() { + local module="${1}" + # Can't use 'kldstat -m foo' as it requires a module file on disk. WTF? + kldstat | grep -q "${module}" +} + +cloudiatr_mount_dist_image() { + # intentionally leaks non-local variable md + + if [ ! -f "${CLOUDIATR_DIST_IMAGE}" ]; then + cloudiatr_wtf "File ${CLOUDIATR_DIST_IMAGE} does not exist" + return 1 + fi + + if "${CLOUDIATR_CHECKSUM_SMECKSUM}"; then + cloudiatr_fyi "Checksum smecksum" + else + cloudiatr_fyi "Checking checksum for image file ${CLOUDIATR_DIST_IMAGE} ..." + sha256 -c "${CLOUDIATR_DIST_IMAGE_SHA256}" "${CLOUDIATR_DIST_IMAGE}" + fi + md=$(mdconfig -f "${CLOUDIATR_DIST_IMAGE}") + if [ -z "${md}" ]; then + return 1 + fi + + for potential_partition in "/dev/${md}a" "/dev/${md}p2"; do + if [ -c "${potential_partition}" ]; then + cloudiatr_fyi "Trying to mount ${potential_partition} ..." + mount -o ro "${potential_partition}" /mnt/ || return 1 + cloudiatr_fyi "Mounting ${potential_partition} worked..." + fi + done +} + +cloudiatr_extract_distribution() { + local chroot_dir \ + md extra_distribution + + chroot_dir="${1}" + + if [ -n "${CLOUDIATR_DIST_IMAGE}" ]; then + cloudiatr_mount_dist_image + else + cloudiatr_fyi "No CLOUDIATR_DIST_IMAGE specified." + cloudiatr_fyi "Using CLOUDIATR_DIST_DIR=${CLOUDIATR_DIST_DIR}!" + fi + + cloudiatr_fyi "Extracting base in ${chroot_dir} ..." + # Exclude /var/empty as it's read-only + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/base.txz" --exclude ./var/empty/) + + cloudiatr_fyi "Creating directories that were missing in the base tarball" + chroot "${chroot_dir}" mtree -f /etc/mtree/BSD.root.dist -d -e -u + chroot "${chroot_dir}" mtree -f /etc/mtree/BSD.var.dist -d -e -u -p var + + for extra_distribution in ${CLOUDIATR_EXTRA_DISTRIBUTIONS}; do + cloudiatr_fyi "Extracting extra distribution '${extra_distribution}'" + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/${extra_distribution}.txz") + done + + cloudiatr_fyi "Extracting kernel (without symbols) in ${chroot_dir} ..." + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/kernel.txz" --exclude "*.symbols") + + if [ -n "${CLOUDIATR_DIST_IMAGE}" ]; then + umount /mnt + mdconfig -d -u ${md##md} + fi +} + +cloudiatr_setup_new_user() { + local ssh_dir="${chroot_dir}/home/${CLOUDIATR_NEW_USER}/.ssh" + + cloudiatr_fyi "Creating user '${CLOUDIATR_NEW_USER}'" + + echo "${CLOUDIATR_INITIAL_PASSWORD}" | chroot "${chroot_dir}" \ + pw useradd "${CLOUDIATR_NEW_USER}" -G wheel,operator -h 0 + mkdir "${ssh_dir}" + cp -v /root/.ssh/authorized_keys "${ssh_dir}" || true + chroot "${chroot_dir}" chown -R "${CLOUDIATR_NEW_USER}" "/home/${CLOUDIATR_NEW_USER}" + chroot "${chroot_dir}" chmod -R go-rwx "/home/${CLOUDIATR_NEW_USER}" +} + +cloudiatr_create_geli_key() { + local keyfile="${1}" + + ( + umask 077 + dd bs=64 count=1 if=/dev/random of="${keyfile}" 2>/dev/null + ) +} + +cloudiatr_setup_geli() { + local disks \ + d + + disks="${*}" + + cloudiatr_create_geli_key "${CLOUDIATR_RPOOL_KEY}" + + mkdir "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups" + + for d in $disks; do + cloudiatr_fyi "Initialising geli on ${d}p3 ..." + geli init -b -B "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups/${d##/dev/}p3.eli" \ + -P -K "${CLOUDIATR_RPOOL_KEY}" -l "${CLOUDIATR_GELI_KEY_LENGTH}" \ + -e "${CLOUDIATR_GELI_EALGO}" -s 4096 -V 7 "${d}p3" > /dev/null + done + + cloudiatr_attach_geli_geoms "${CLOUDIATR_RPOOL_KEY}" "p3" +} + +cloudiatr_attach_geli_geoms() { + local keyfile partition_id \ + disk + + keyfile="${1}" + partition_id="${2}" + + for disk in ${CLOUDIATR_DISKS}; do + cloudiatr_fyi "geli-attaching ${disk}${partition_id}" + geli attach -p -k "$keyfile" "${disk}${partition_id}" + done +} + +cloudiatr_detach_geli_geoms() { + local partition_id + + partition_id="${1}" + + for disk in ${CLOUDIATR_DISKS}; do + cloudiatr_fyi "geli-detaching ${disk}${partition_id}" + geli detach "${disk}${partition_id}.eli" + done +} + +cloudiatr_get_bpool_geoms() { + cloudiatr_get_geoms p2 +} + +cloudiatr_get_disk_names() { + local disk + for disk in ${CLOUDIATR_DISKS}; do + echo "${disk##*/}" + done +} + +cloudiatr_get_number_of_disks() { + local \ + disk number_of_disks + + number_of_disks=0 + for disk in ${CLOUDIATR_DISKS}; do + number_of_disks=$((number_of_disks+1)) + done + echo "${number_of_disks}" +} + +cloudiatr_get_default_pool_layout() { + if [ "$(cloudiatr_get_number_of_disks)" = 1 ]; then + echo "single-disk" + else + echo "mirror" + fi +} + +cloudiatr_autodetect_disks() { + local \ + disk + + for disk in $(sysctl -n kern.disks); do + # Only use ada(4) devices. We obviously can't use cd(4) + # devices and using da(4) devices would require us to + # skip the one we (probably) booted from. + if [ "${disk##ada}" != "${disk}" ]; then + echo "/dev/${disk}" + fi + done +} + +cloudiatr_create_bpool() { + local \ + bpool_geoms pool_layout + + bpool_geoms="$(cloudiatr_get_bpool_geoms)" + pool_layout="${CLOUDIATR_BPOOL_LAYOUT}" + + if [ "${pool_layout}" = "default" ]; then + pool_layout="$(cloudiatr_get_default_pool_layout)" + fi + + cloudiatr_fyi "Creating boot pool '${CLOUDIATR_BPOOL_NAME}' on ${bpool_geoms}. Pool layout: ${pool_layout}" + + zpool create -f -o version=28 -O compression=lzjb \ + "${CLOUDIATR_BPOOL_NAME}" ${pool_layout##single-disk} \ + $bpool_geoms + + # This currently can't be set at create-time + zpool set "bootfs=${CLOUDIATR_BPOOL_NAME}" "${CLOUDIATR_BPOOL_NAME}" + + # Would be nice, but for the bootfs to work, + # its ./boot directory can't be a zfs fs. + # + # XXX: can we work around this by setting bootfs + # on bpool/boot and use a symlink from bootf/boot/boot + # to bootf/boot? + # zfs create "${CLOUDIATR_BPOOL_NAME}/boot" +} + +cloudiatr_setup_tmpfs() { + mkdir -p "${CLOUDIATR_NEW_SYSTEM_DIR}" + mount -t tmpfs tmpfs "${CLOUDIATR_NEW_SYSTEM_DIR}" +} + +cloudiatr_generate_rc_conf() { + local \ + netmask + + cat < "${config_file}" +} + +cloudiatr_create_config_files() { + local chroot_dir="${1}" + + cloudiatr_generate_file loader_conf "${chroot_dir}/boot/loader.conf" + cloudiatr_generate_file sysctl_conf "${chroot_dir}/etc/sysctl.conf" + cloudiatr_generate_file rc_conf "${chroot_dir}/etc/rc.conf" + cloudiatr_generate_file fstab "${chroot_dir}/etc/fstab" + cloudiatr_generate_file resolv_conf "${chroot_dir}/etc/resolv.conf" || true +} + +cloudiatr_get_required_kernel_content() { + kldstat | awk '/k/ {print $5}' +} + +# XXX: May creates output with duplicated slashes. Ugly but harmless. +# XXX: Why do we ignore errors here? +cloudiatr_populate_bpool() { + local boot_dir \ + boot_file new_kernel_dir sub_dir bpool_mountpoint new_file + + boot_dir="${1}" + if [ "${boot_dir}" = "/" ]; then + # Prevent duplicated leading slash in log messages + boot_dir="" + fi + bpool_mountpoint="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_BPOOL_NAME}" + + cloudiatr_fyi "Populating boot pool '${CLOUDIATR_BPOOL_NAME}' ..." + zfs set mountpoint="${bpool_mountpoint}" "${CLOUDIATR_BPOOL_NAME}" || return 1 + + # Only copy what we really need before we can import the encrypted rpool + for sub_dir in dtb firmware kernel zfs modules defaults; do + mkdir -p "${bpool_mountpoint}/boot/${sub_dir}" + done + + for boot_file in $(cloudiatr_get_required_kernel_content); do + # XXX: Create missing directories here + cp -v "${boot_dir}/boot/kernel/${boot_file}" "${bpool_mountpoint}/boot/kernel/" || true + done + + for boot_file in $(find "${boot_dir}/boot/" \ + -not -path "${boot_dir}/boot/kernel*" -a \ + -not -path "${boot_dir}/boot/boot*" -a \ + -not -path "${boot_dir}/boot/cdboot" -a \ + -not -path "${boot_dir}/boot/*mbr" -a \ + -type f); do + new_file="${bpool_mountpoint}/boot/${boot_file##*/boot/}" + cp -v "${boot_file}" "${new_file}" || true + done + + zfs set "mountpoint=/${CLOUDIATR_BPOOL_NAME}" "${CLOUDIATR_BPOOL_NAME}" +} + +cloudiatr_clean_up() { + cloudiatr_fyi "Exporting boot pool '${CLOUDIATR_BPOOL_NAME}' ..." + zpool export "${CLOUDIATR_BPOOL_NAME}" + cloudiatr_fyi "Exporting root pool '${CLOUDIATR_RPOOL_NAME}' ..." + zpool export "${CLOUDIATR_RPOOL_NAME}" + cloudiatr_detach_geli_geoms "p3" + umount "${CLOUDIATR_NEW_SYSTEM_DIR}" + rmdir "${CLOUDIATR_NEW_SYSTEM_DIR}" + if "${CLOUDIATR_USE_SWAP_WHILE_INSTALLING}"; then + if ! cloudiatr_disable_swap ${CLOUDIATR_DISKS}; then + cloudiatr_fyi "Failed to remove all the added swap space." + cloudiatr_fyi "If the system is low on memory the problem can be safely ignored." + fi + fi +} + +cloudiatr_generate_ssh_hostkeys() { + local chroot_dir \ + real_hostname + + chroot_dir="${1}" + real_hostname="$(hostname)" + + hostname "${CLOUDIATR_HOSTNAME}" + cloudiatr_fyi "Generating ssh host keys for ${CLOUDIATR_HOSTNAME} ..." + for key_alg in ${CLOUDIATR_SSHD_HOST_KEY_ALGORITHMS}; do + key_file="${chroot_dir}/etc/ssh/ssh_host_${key_alg}_key" + if ! ssh-keygen -q -t "${key_alg}" -f "${key_file}" -N ""; then + if [ "${key_alg}" = "ed25519" ]; then + # ed25519 isn't supported on FreeBSD 10.0 and earlier, + # thus we allow this to fail + continue + fi + return 1 + fi + ssh-keygen -l -v -f "${key_file}.pub" + done + hostname "${real_hostname}" +} + +cloudiatr_collect_evidence() { + local \ + evidence_dataset evidence disk_name + + evidence_dataset="${CLOUDIATR_RPOOL_NAME}/cloudiatr-evidence" + + cloudiatr_fyi "Collecting 'evidence' in /${evidence_dataset} ..." + zfs create "${evidence_dataset}" + for evidence in "${CLOUDIATR_CONFIG_FILE}" "${0}" \ + "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups/"* \ + "${CLOUDIATR_RPOOL_KEY}"; do + cp -p "${evidence}" "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}" + done + + for disk_name in $(cloudiatr_get_disk_names); do + evidence="${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}/${disk_name}.gpart" + gpart backup "${disk_name}" > "${evidence}" + done + + if "${CLOUDIATR_SAVE_DIST_DIR}"; then + cp -r "${CLOUDIATR_DIST_DIR%%/}" "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}/" + fi + + find "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}" -type f | sort +} + +cloudiatr_generate_sshd_config_extension() { + cat <> ${chroot_dir}/etc/ssh/sshd_config + + if "${CLOUDIATR_USE_NTPDATE}"; then + chroot ${chroot_dir} service ntpdate onestart || true + fi + + umount ${chroot_dir}/dev/ + + cloudiatr_setup_new_user + + # Copying the zpool.cache is no longer necessary on ElectroBSD + # and recent FreeBSD versions, but doesn't hurt. + cp /boot/zfs/zpool.cache "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/boot/zfs/zpool.cache" + + cloudiatr_populate_bpool "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}" + + cloudiatr_collect_evidence + + cloudiatr_fyi "Setting final mountpoints on root pool '${CLOUDIATR_RPOOL_NAME}' ..." + zfs umount "${CLOUDIATR_RPOOL_NAME}" + zfs set mountpoint=legacy "${CLOUDIATR_RPOOL_NAME}" + for fs in boot home tmp usr var; do + zfs set "mountpoint=/${fs}" "${CLOUDIATR_RPOOL_NAME}/${fs}" + done + + cloudiatr_clean_up +} + +# The murder-death-kill feature was added for testing. After the introduction of +# the boring "clean-up" subcommand (which doesn't involve killing) it could be +# considered obsolete, but keeping it around makes cloudiatr more awesome. +cloudiatr_murder_death_kill() { + if "${CLOUDIATR_MURDER_DEATH_KILL_REQUESTED}"; then + cloudiatr_fyi "You really asked for it. Murder death kill in progress ..." + zpool export "${CLOUDIATR_BPOOL_NAME}" || true + zpool export "${CLOUDIATR_RPOOL_NAME}" || true + umount "${CLOUDIATR_NEW_SYSTEM_DIR}" || true + geli kill -a || true + fi +} + +cloudiatr_request_consent() { + local message="${*}" \ + response + + if "${CLOUDIATR_DONT_ASK_JUST_KISS}"; then + # ... the data goodbye. + return 0 + fi + + echo -n "cloudiatr: ${message} [y/n] " + # XXX: Don't use "read -p" as it may work unreliably + read response + + # "No" means "no". Everything but "y" also means "no". + [ "${response}" = "y" ] +} + +cloudiatr_has_eviction_consent() { + echo "cloudiatr (${CLOUDIATR_VERSION}) can't wait to evict '$(hostname)' ..." + echo + echo "Depending on your jurisdiction, 'eviction without consent' may be against the law." + echo "cloudiatr doesn't bother to make backups of the existing data. That's what zogftw is for." + echo + cloudiatr_request_consent "Continue eviction?" +} + +cloudiatr_evict() { + if cloudiatr_has_eviction_consent; then + if "${CLOUDIATR_MURDER_DEATH_KILL_REQUESTED}"; then + cloudiatr_murder_death_kill + else + cloudiatr_fyi "You asked for it ..." + fi + cloudiatr_fyi "Eviction in progress ..." + cloudiatr_evict_local_system + cloudiatr_fyi "Looks like somebody managed to install a real operating system ..." + if cloudiatr_request_consent "Reboot now?"; then + shutdown -r now + fi + return 0 + else + cloudiatr_fyi "Eviction aborted in time ..." + return 1 + fi +} + +cloudiatr_has_soft_protect_consent() { + cloudiatr_request_consent "Put $(uname) in 'Soviet Germany' mode?" +} + +# XXX: Only works for the cloudiatr disk layout. +cloudiatr_soft_protect() { + local \ + mirror_name device_to_clear number_of_disks + + mirror_name="vdev-remains" + number_of_disks="$(cloudiatr_get_number_of_disks)" + + cloudiatr_fyi "Destroying ${CLOUDIATR_BPOOL_NAME} ..." + cloudiatr_fyi "Use 'geli kill -a' to 'hard-protect' your data right now. No recovery without remote backups!" + + if zpool list "${CLOUDIATR_BPOOL_NAME}" >/dev/null 2>&1; then + zpool destroy "${CLOUDIATR_BPOOL_NAME}" || true + fi + + if [ "${number_of_disks}" = 1 ]; then + cloudiatr_fyi "Nuking former ${CLOUDIATR_BPOOL_NAME} vdev from orbit ..." + device_to_clear="$(cloudiatr_get_bpool_geoms)" + else + cloudiatr_fyi "Nuking former ${CLOUDIATR_BPOOL_NAME} vdevs from orbit using gmirror power ..." + gmirror load 2>/dev/null || true + gmirror label "${mirror_name}" $(cloudiatr_get_bpool_geoms) + device_to_clear="/dev/mirror/${mirror_name}" + fi + + geli onetime -s 4096 -e "${CLOUDIATR_GELI_EALGO}" "${device_to_clear}" + dd if=/dev/zero bs=1M of="${device_to_clear}.eli" 2>/dev/null || true + geli detach "${device_to_clear##/dev/}" + + cloudiatr_fyi "Done. Levelling nuked wasteland with zeroes ..." + dd if=/dev/zero bs=1M of="${device_to_clear}" 2>/dev/null || true + + if [ "${number_of_disks}" != 1 ]; then + gmirror destroy "${mirror_name}" + fi + cloudiatr_fyi "Done. $(uname) should remain working as expected until the next shutdown ..." + cloudiatr_fyi "Remember to 'unprotect' the system before consensual reboots (or use the opportunity to test your backup system)" +} + +cloudiatr_check_privileges() { + local \ + uid user + + uid="$(id -u)" + user="$(id -un)" + + if [ "${uid}" != 0 ]; then + cloudiatr_wtf "Check your privileges, $user. It looks like you might not have enough of them!" + cloudiatr_fyi "Hint: Using 'sudo' or 'su' might help." + return 1 + fi + + return 0 +} + +cloudiatr_usage() { + local \ + subcommand + + for subcommand in clean-up cmd recreate-bpool evict soft-protect show-config; do + echo "cloudiatr [-f config-file] $subcommand" + done +} + +cloudiatr_update_base() { + local \ + base_dist="${CLOUDIATR_DIST_DIR}/base.txz" + + if [ ! -f "${base_dist}" ]; then + cloudiatr_wtf "Base update impossible. '${base_dist}' does not exist" + return 1 + fi + cloudiatr_wtf "Updating base system ..." + chflags -R noschg /bin/ /lib/ /sbin/ /usr/bin/ /usr/lib /libexec/ || return 1 + cd / || return 1 + tar xpf "${base_dist}" --exclude ./etc/ --exclude ./var/empty || return 1 +} + +cloudiatr_main() { + local mode + + if [ "${1}" = "-f" ]; then + shift + CLOUDIATR_CONFIG_FILE="${1}" + shift + if [ -z "${CLOUDIATR_CONFIG_FILE}" ] || + ! [ -f "${CLOUDIATR_CONFIG_FILE}" ]; then + cloudiatr_wtf "No existing config file specified." + return 1 + fi + fi + + mode="${1}" + + if [ "${mode}" != "cmd" ] && [ "${mode}" != "show-config" ]; then + cloudiatr_check_privileges || return 1 + fi + + cloudiatr_init "${mode}" + + case "${mode}" in + clean-up) + set +e + cloudiatr_clean_up + ;; + cmd) + shift + "${@}" + ;; + recreate-bpool) + cloudiatr_create_bpool + cloudiatr_populate_bpool "/" + ;; + evict) + cloudiatr_evict + ;; + soft-protect) + if cloudiatr_has_soft_protect_consent; then + cloudiatr_soft_protect + fi + ;; + update) + cloudiatr_update_base || return 1 + ;; + show-config) + cloudiatr_show_config + ;; + #rekey) + # XXX: Not yet implemented + *) + cloudiatr_usage + cloudiatr_wtf "Invalid or missing subcommand" + return 1 + ;; + esac + +} + +cloudiatr_main "${@}" -- 2.7.0 From c3b7369945305d22d8b493640f509e8a1981d72d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 Jan 2016 12:49:29 +0100 Subject: [PATCH 207/213] OpenSSH: Disable DSA keys by default ... as it's done by vanilla OpenSSH since 6.9p1. This reverts FreeBSD r294495 which enabled them again due to POLA concerns that aren't relevant for ElectroBSD. --- UPDATING | 4 ++++ crypto/openssh/myproposal.h | 5 +---- crypto/openssh/ssh_config.5 | 18 ++++++------------ crypto/openssh/sshd_config.5 | 18 ++++++------------ 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/UPDATING b/UPDATING index 123c05a..6fa836c 100644 --- a/UPDATING +++ b/UPDATING @@ -39,6 +39,10 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 11.x IS SLOW: restored by setting vfs.zfs.vol.recursive=1. 20160119: + The default configuration of ssh(1) no longer allows to use ssh-dss + keys. To enable using them, add 'ssh-dss' to PubkeyAcceptedKeyTypes + option in the /etc/ssh/ssh_config. Refer to ssh_config(5) for more + information. The NONE and HPN patches has been removed from OpenSSH. They are still available in the security/openssh-portable port. diff --git a/crypto/openssh/myproposal.h b/crypto/openssh/myproposal.h index 83fc943..46e5b98 100644 --- a/crypto/openssh/myproposal.h +++ b/crypto/openssh/myproposal.h @@ -1,5 +1,4 @@ /* $OpenBSD: myproposal.h,v 1.47 2015/07/10 06:21:53 markus Exp $ */ -/* $FreeBSD$ */ /* * Copyright (c) 2000 Markus Friedl. All rights reserved. @@ -100,11 +99,9 @@ HOSTKEY_ECDSA_CERT_METHODS \ "ssh-ed25519-cert-v01@openssh.com," \ "ssh-rsa-cert-v01@openssh.com," \ - "ssh-dss-cert-v01@openssh.com," \ HOSTKEY_ECDSA_METHODS \ "ssh-ed25519," \ - "ssh-rsa," \ - "ssh-dss" + "ssh-rsa" \ /* the actual algorithms */ diff --git a/crypto/openssh/ssh_config.5 b/crypto/openssh/ssh_config.5 index 9f67608..a461238 100644 --- a/crypto/openssh/ssh_config.5 +++ b/crypto/openssh/ssh_config.5 @@ -798,10 +798,8 @@ ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, -ssh-dss-cert-v01@openssh.com, -ecdsa-sha2-nistp256,ecdsa-sha2-nistp384, -ecdsa-sha2-nistp521,ssh-ed25519, -ssh-rsa,ssh-dss +ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, +ssh-ed25519,ssh-rsa .Ed .Pp The @@ -823,10 +821,8 @@ ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, -ssh-dss-cert-v01@openssh.com, -ecdsa-sha2-nistp256,ecdsa-sha2-nistp384, -ecdsa-sha2-nistp521,ssh-ed25519, -ssh-rsa,ssh-dss +ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, +ssh-ed25519,ssh-rsa .Ed .Pp If hostkeys are known for the destination host then this default is modified @@ -1255,10 +1251,8 @@ ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, -ssh-dss-cert-v01@openssh.com, -ecdsa-sha2-nistp256,ecdsa-sha2-nistp384, -ecdsa-sha2-nistp521,ssh-ed25519, -ssh-rsa,ssh-dss +ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, +ssh-ed25519,ssh-rsa .Ed .Pp The diff --git a/crypto/openssh/sshd_config.5 b/crypto/openssh/sshd_config.5 index 9945fc9..2a4a551 100644 --- a/crypto/openssh/sshd_config.5 +++ b/crypto/openssh/sshd_config.5 @@ -657,10 +657,8 @@ ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, -ssh-dss-cert-v01@openssh.com, -ecdsa-sha2-nistp256,ecdsa-sha2-nistp384, -ecdsa-sha2-nistp521,ssh-ed25519, -ssh-rsa,ssh-dss +ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, +ssh-ed25519,ssh-rsa .Ed .Pp The @@ -754,10 +752,8 @@ ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, -ssh-dss-cert-v01@openssh.com, -ecdsa-sha2-nistp256,ecdsa-sha2-nistp384, -ecdsa-sha2-nistp521,ssh-ed25519, -ssh-rsa,ssh-dss +ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, +ssh-ed25519,ssh-rsa .Ed .Pp The list of available key types may also be obtained using the @@ -1359,10 +1355,8 @@ ecdsa-sha2-nistp384-cert-v01@openssh.com, ecdsa-sha2-nistp521-cert-v01@openssh.com, ssh-ed25519-cert-v01@openssh.com, ssh-rsa-cert-v01@openssh.com, -ssh-dss-cert-v01@openssh.com, -ecdsa-sha2-nistp256,ecdsa-sha2-nistp384, -ecdsa-sha2-nistp521,ssh-ed25519, -ssh-rsa,ssh-dss +ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521, +ssh-ed25519,ssh-rsa .Ed .Pp The -- 2.7.0 From 15e40f9418916942cbdd4633fbe19806b6d70a5e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 Jan 2016 19:57:05 +0100 Subject: [PATCH 208/213] reproduce.sh: Build kernel after world to make sure the toolchain is fresh --- reproduce.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index 4fbb77c..034d5a7 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -71,12 +71,14 @@ reproduce_all_the_things() { export MAKEOBJDIRPREFIX announce_status "MAKEOBJDIRPREFIX is set to ${MAKEOBJDIRPREFIX}" - announce_status "Starting to build the kernel" - make buildkernel || return 1 - + # We build the world first, so the kernel is built + # with a freshly built toolchain. announce_status "Starting to build the world" make -j${MAX_MAKE_JOBS} buildworld || return 1 + announce_status "Starting to build the kernel" + make buildkernel || return 1 + # Make sure obj files aren't dumped in ${SRC_DIR} mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 -- 2.7.0 From 1317144a8cf99589c0a1c807032b283da03a02f8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 Jan 2016 13:32:24 +0100 Subject: [PATCH 209/213] ZFS ARC: If there's less than 1 GB of RAM, set the ARC max to 50% of it Now the code matches the comment again. The limit was increased to 6/8 in r172130 and reduced to 5/8 in r175633 again to mitigate 'kmem_map too small panics'. As it turns out, 5/8 is still too much and can result in deadlocks caused by the vm system running out of pages: (kgdb) p vm_cnt $7 = {v_swtch = 0, v_trap = 0, v_syscall = 0, v_intr = 0, v_soft = 0, v_vm_faults = 0, v_io_faults = 0, v_cow_faults = 0, v_cow_optim = 0, v_zfod = 0, v_ozfod = 0, v_swapin = 0, v_swapout = 0, v_swappgsin = 0, v_swappgsout = 0, v_vnodein = 0, v_vnodeout = 0, v_vnodepgsin = 0, v_vnodepgsout = 0, v_intrans = 0, v_reactivated = 0, v_pdwakeups = 878, v_pdpages = 0, v_tcached = 0, v_dfree = 0, v_pfree = 0, v_tfree = 0, v_page_size = 4096, v_page_count = 247933, v_free_reserved = 372, v_free_target = 5324, v_free_min = 1610, v_free_count = 2, v_wire_count = 72901, v_active_count = 174634, v_inactive_target = 7986, v_inactive_count = 395, v_cache_count = 0, v_pageout_free_min = 34, v_interrupt_free_min = 2, v_free_severe = 991, v_forks = 0, v_vforks = 0, v_rforks = 0, v_kthreads = 0, v_forkpages = 0, v_vforkpages = 0, v_rforkpages = 0, v_kthreadpages = 0, v_spare = 0xffffffff8141770c} --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 550a3a7..9068dd6 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -5445,7 +5445,7 @@ arc_init(void) arc_c_max = (arc_c * 8) - (1 << 30); else arc_c_max = arc_c_min; - arc_c_max = MAX(arc_c * 5, arc_c_max); + arc_c_max = MAX(arc_c * 4, arc_c_max); /* * In userland, there's only the memory pressure that we artificially -- 2.7.0 From e6bbc6aa9ffade1703664abd144a2392787ce784 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 15 Jan 2016 18:35:29 +0100 Subject: [PATCH 210/213] reproduce.sh: Move created files to their own directory when done --- reproduce.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/reproduce.sh b/reproduce.sh index 034d5a7..cfca31c 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -62,6 +62,8 @@ announce_status() { } reproduce_all_the_things() { + local \ + f release_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/release" if "${RESUME_BUILD}"; then announce_status "Resuming ..." @@ -80,7 +82,7 @@ reproduce_all_the_things() { make buildkernel || return 1 # Make sure obj files aren't dumped in ${SRC_DIR} - mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 + mkdir -p "${release_dir}" || return 1 if ! "${RESUME_BUILD}"; then announce_status "Starting to clean the release dir" @@ -90,6 +92,16 @@ reproduce_all_the_things() { time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 announce_status "Done with release memstick for ${REPRO_SEED}" + + cd "${release_dir}" || return 1 + mkdir "${BUILD}" || return 1 + for f in *.txz MANIFEST; do + mv "${f}" "${BUILD}/" || return + done + mv memstick.img "${BUILD}/${BUILD}.img" || return 1 + + announce_status "Created files copied to ${release_dir}/${BUILD}" + sha256 "${release_dir}/${BUILD}"/* } assert_untainted_source_tree() { -- 2.7.0 From b70213fcbbd3bf69ea5d52bf55b4b3368115ed2a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 Jan 2016 12:44:01 +0100 Subject: [PATCH 211/213] ZFS ARC: Disable the 'WAKE_PAGER' ... as I suspect that the "Don't do it if we woke the pager" code in arc_lowmem() increase the chances that the vm runs out of free pages. It's also not obvious (to me) that the WAKE_PAGER does anything useful. Quoting https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594#c244: "I have not yet tried Karl's latest patch but like to point out that some of the underlying assumptions about how the vm pager behaves seem incorrect to me. For example I'd expect calling pagedaemon_wakeup() without memory pressure (from the pagers point of view) to be pretty close to a nop as vm_pageout_worker() does its own checks before doing any heavy lifting: http://fxr.watson.org/fxr/source/vm/vm_pageout.c#L1634 Also note that vm_pageout_worker() is already called at least once per second anyway: [fk@polizei-erziehung ~]$ sudo /usr/src/share/dtrace/monitor-page-scanner 2015 Nov 1 17:20:45: Monitoring the page scanner. Minimum pass value to show 'boring' scans without memory pressure or inactive page surplus: 2 (Launder dirty pages). Press CTRL-C to abort. 2015 Nov 1 17:21:45: Scan goals in the previous minute: Update active LRU/deactivate pages 60 2015 Nov 1 17:22:45: Scan goals in the previous minute: Update active LRU/deactivate pages 60 I'm not claiming that increasing the frequency when there's no memory pressure causes any harm (besides code complexity), but I'm not convinced that it has the intended effect and needs to be triggered from ZFS (as opposed to changing the pager defaults)." --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 9068dd6..131bb11 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -245,7 +245,7 @@ u_int zfs_arc_free_target = 0; u_int zfs_arc_wakeup_pager = 0; u_int zfs_arc_wakeup_delay = 500; -#define WAKE_PAGER +#undef WAKE_PAGER #ifdef WAKE_PAGER #define WAKE_PAGER_CONSTANT 10 / 9 /* Pager wakeup threshold */ static int arc_init_done = 0; /* We know arc_warm is valid */ -- 2.7.0 From 3857a99130d02025e21efc1a8562f24b28c3cb6c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 2 Feb 2016 20:08:26 +0100 Subject: [PATCH 212/213] makefs: Import patch to add -T option to clamp timestamps ... from Debian GNU/kFreeBSD. Patch Author: Steven Chamberlain Patch source: http://sources.debian.net/patches/patch/makefs/20100306-6/maxtime-option.diff/ --- usr.sbin/makefs/ffs.c | 21 +++++++++++++++++++++ usr.sbin/makefs/makefs.8 | 5 +++++ usr.sbin/makefs/makefs.c | 16 +++++++++++++--- usr.sbin/makefs/makefs.h | 1 + 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/usr.sbin/makefs/ffs.c b/usr.sbin/makefs/ffs.c index dd97c01..aa2d335 100644 --- a/usr.sbin/makefs/ffs.c +++ b/usr.sbin/makefs/ffs.c @@ -661,6 +661,27 @@ ffs_build_dinode1(struct ufs1_dinode *dinp, dirbuf_t *dbufp, fsnode *cur, dinp->di_mtimensec = cur->inode->st.st_mtimensec; dinp->di_ctimensec = cur->inode->st.st_ctimensec; #endif + /* if maxtime was given, clamp all timestamps to this */ + if (fsopts->maxtime >= 0) { + if (dinp->di_atime >= fsopts->maxtime) { + dinp->di_atime = fsopts->maxtime; +#if HAVE_STRUCT_STAT_ST_MTIMENSEC + dinp->di_atimensec = 0; +#endif + } + if (dinp->di_mtime >= fsopts->maxtime) { + dinp->di_mtime = fsopts->maxtime; +#if HAVE_STRUCT_STAT_ST_MTIMENSEC + dinp->di_mtimensec = 0; +#endif + } + if (dinp->di_ctime >= fsopts->maxtime) { + dinp->di_ctime = fsopts->maxtime; +#if HAVE_STRUCT_STAT_ST_MTIMENSEC + dinp->di_ctimensec = 0; +#endif + } + } #if HAVE_STRUCT_STAT_ST_FLAGS dinp->di_flags = cur->inode->st.st_flags; #endif diff --git a/usr.sbin/makefs/makefs.8 b/usr.sbin/makefs/makefs.8 index f80dc53..4834f1f 100644 --- a/usr.sbin/makefs/makefs.8 +++ b/usr.sbin/makefs/makefs.8 @@ -57,6 +57,7 @@ .Op Fl S Ar sector-size .Op Fl s Ar image-size .Op Fl t Ar fs-type +.Op Fl T Ar maximum-time .Ar image-file .Ar directory | manifest .Op Ar extra-directory ... @@ -223,6 +224,10 @@ BSD fast file system (default). .It Sy cd9660 ISO 9660 file system. .El +.It Fl T Ar maximum-time +Clamp superblock and file timestamps to +.Ar maximum-time +seconds since the Epoch. .It Fl x Exclude file system nodes not explicitly listed in the specfile. .It Fl Z diff --git a/usr.sbin/makefs/makefs.c b/usr.sbin/makefs/makefs.c index a7ca751..047b2dd 100644 --- a/usr.sbin/makefs/makefs.c +++ b/usr.sbin/makefs/makefs.c @@ -105,6 +105,7 @@ main(int argc, char *argv[]) (void)memset(&fsoptions, 0, sizeof(fsoptions)); fsoptions.fd = -1; fsoptions.sectorsize = -1; + fsoptions.maxtime = -1; if (fstype->prepare_options) fstype->prepare_options(&fsoptions); @@ -116,7 +117,7 @@ main(int argc, char *argv[]) start_time.tv_sec = start.tv_sec; start_time.tv_nsec = start.tv_usec * 1000; - while ((ch = getopt(argc, argv, "B:b:Dd:f:F:M:m:N:o:pR:s:S:t:xZ")) != -1) { + while ((ch = getopt(argc, argv, "B:b:Dd:f:F:M:m:N:o:pR:s:S:t:T:xZ")) != -1) { switch (ch) { case 'B': @@ -239,6 +240,14 @@ main(int argc, char *argv[]) fstype->prepare_options(&fsoptions); break; + case 'T': + fsoptions.maxtime = strtoll(optarg, NULL, 10); + if (start_time.tv_sec >= fsoptions.maxtime) { + start_time.tv_sec = fsoptions.maxtime; + start_time.tv_nsec = 0; + } + break; + case 'x': fsoptions.onlyspec = 1; break; @@ -369,8 +378,9 @@ usage(void) fprintf(stderr, "usage: %s [-t fs-type] [-o fs-options] [-d debug-mask] [-B endian]\n" "\t[-S sector-size] [-M minimum-size] [-m maximum-size] [-R roundup-size]\n" -"\t[-s image-size] [-b free-blocks] [-f free-files] [-F mtree-specfile]\n" -"\t[-xZ] [-N userdb-dir] image-file directory | manifest [extra-directory ...]\n", +"\t[-s image-size] [-T maximum-time] [-b free-blocks] [-f free-files]\n" +"\t[-F mtree-specfile] [-xZ] [-N userdb-dir]\n" +"\timage-file directory | manifest [extra-directory ...]\n", prog); exit(1); } diff --git a/usr.sbin/makefs/makefs.h b/usr.sbin/makefs/makefs.h index ba80f74..a1df23e 100644 --- a/usr.sbin/makefs/makefs.h +++ b/usr.sbin/makefs/makefs.h @@ -123,6 +123,7 @@ typedef struct { /* global options */ off_t minsize; /* minimum size image should be */ off_t maxsize; /* maximum size image can be */ + time_t maxtime; /* maximum allowed timestamp value */ off_t freefiles; /* free file entries to leave */ int freefilepc; /* free file % */ off_t freeblocks; /* free blocks to leave */ -- 2.7.0 From 0c7f4d63d7bbf42f717ddd98dff53cb128179fe0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 2 Feb 2016 20:28:35 +0100 Subject: [PATCH 213/213] release/amd64/make-memstick.sh: Use the shiny new -T switch to clamp the remaing timestamps --- release/amd64/make-memstick.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 66ee073..84d58a5 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -37,7 +37,11 @@ echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local mtree -c -k time -p "${1}" | sed \ -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ > "${2}.mtree" || return 1 -makefs -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} + +# The base mtree may not have -T support yet, +# thus we use the fresh one that goes into the image. +dist/base/usr/sbin/makefs -T "${EPOCH_DATE-0}" \ + -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 -- 2.7.0